From 9ff66aac08cd4f5448c98d7951174e3d4e23b4eb Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Wed, 7 May 2025 09:49:34 +0530 Subject: [PATCH 1/8] HDDS-12776. CLI command to list all Duplicate open containers --- .../debug/logs/container/ListContainers.java | 15 +++++-- .../utils/ContainerDatanodeDatabase.java | 39 +++++++++++++++++++ .../logs/container/utils/SQLDBConstants.java | 4 ++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java index 6bc2e4b13641..126eaf7bda16 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java @@ -37,13 +37,16 @@ public class ListContainers extends AbstractSubcommand implements Callable { @CommandLine.Option(names = {"--state"}, - description = "Life cycle state of the container.", - required = true) + description = "Life cycle state of the container.") private HddsProtos.LifeCycleState state; @CommandLine.Mixin private ListOptions listOptions; + @CommandLine.Option(names = {"--double-open"}, + description = "List all the containers which have duplicate open states.") + private boolean doubleOpen; + @CommandLine.ParentCommand private ContainerLogController parent; @@ -57,7 +60,13 @@ public Void call() throws Exception { ContainerDatanodeDatabase cdd = new ContainerDatanodeDatabase(dbPath.toString()); - cdd.listContainersByState(state.name(), listOptions.getLimit()); + if (doubleOpen) { + cdd.findDoubleOpenContainer(); + } else if (state != null) { + cdd.listContainersByState(state.name(), listOptions.getLimit()); + } else { + err().println("Please provide either a container state or use --double-open."); + } return null; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java index e05f70e7bb31..76d0e6ed1249 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java @@ -541,5 +541,44 @@ private List getContainerLogData(Long containerID, Connec return logEntries; } + + private void createIdxContainerlogContainerId(Connection conn) throws SQLException { + String sql = SQLDBConstants.CREATE_CONTAINER_ID_INDEX; + try (Statement stmt = conn.createStatement()) { + stmt.execute(sql); + } + } + + public void findDoubleOpenContainer() throws SQLException { + String sql = SQLDBConstants.SELECT_DISTINCT_CONTAINER_IDS_QUERY; + + try (Connection connection = getConnection()) { + + createIdxContainerlogContainerId(connection); + + try (PreparedStatement statement = connection.prepareStatement(sql); + ResultSet resultSet = statement.executeQuery()) { + int count = 0; + + while (resultSet.next()) { + Long containerID = resultSet.getLong("container_id"); + List logEntries = getContainerLogData(containerID, connection); + logEntries.sort(Comparator.comparing(DatanodeContainerInfo::getTimestamp)); + boolean hasIssue = checkForMultipleOpenStates(logEntries); + if (hasIssue) { + count++; + out.println("Container ID: " + containerID); + } + } + + out.println("Total containers that might have duplicate OPEN state : " + count); + + } + } catch (SQLException e) { + throw new SQLException("Error while retrieving containers."); + } catch (Exception e) { + throw new RuntimeException("Unexpected error: " + e); + } + } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java index c58b605a2f64..67b5fa181aa9 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java @@ -68,6 +68,10 @@ public final class SQLDBConstants { "WHERE d.container_id = ? ORDER BY d.datanode_id ASC, d.timestamp ASC;"; public static final String CREATE_DCL_CONTAINER_STATE_TIME_INDEX = "CREATE INDEX IF NOT EXISTS " + "idx_dcl_container_state_time ON DatanodeContainerLogTable(container_id, container_state, timestamp);"; + public static final String CREATE_CONTAINER_ID_INDEX = "CREATE INDEX IF NOT EXISTS idx_containerlog_container_id " + + "ON ContainerLogTable(container_id);"; + public static final String SELECT_DISTINCT_CONTAINER_IDS_QUERY = + "SELECT DISTINCT container_id FROM ContainerLogTable"; private SQLDBConstants() { //Never constructed From 9f45c0d19a958862315761d61fbabce8c8313612 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Wed, 7 May 2025 11:43:06 +0530 Subject: [PATCH 2/8] Renamed command for listing containers with duplicate open states --- .../hadoop/ozone/debug/logs/container/ListContainers.java | 8 ++++---- .../logs/container/utils/ContainerDatanodeDatabase.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java index 126eaf7bda16..0d8819a8efaa 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java @@ -43,9 +43,9 @@ public class ListContainers extends AbstractSubcommand implements Callable @CommandLine.Mixin private ListOptions listOptions; - @CommandLine.Option(names = {"--double-open"}, + @CommandLine.Option(names = {"--duplicate-open"}, description = "List all the containers which have duplicate open states.") - private boolean doubleOpen; + private boolean duplicateOpen; @CommandLine.ParentCommand private ContainerLogController parent; @@ -60,8 +60,8 @@ public Void call() throws Exception { ContainerDatanodeDatabase cdd = new ContainerDatanodeDatabase(dbPath.toString()); - if (doubleOpen) { - cdd.findDoubleOpenContainer(); + if (duplicateOpen) { + cdd.findDuplicateOpenContainer(); } else if (state != null) { cdd.listContainersByState(state.name(), listOptions.getLimit()); } else { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java index 76d0e6ed1249..f72446ba19f5 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java @@ -549,7 +549,7 @@ private void createIdxContainerlogContainerId(Connection conn) throws SQLExcepti } } - public void findDoubleOpenContainer() throws SQLException { + public void findDuplicateOpenContainer() throws SQLException { String sql = SQLDBConstants.SELECT_DISTINCT_CONTAINER_IDS_QUERY; try (Connection connection = getConnection()) { From a3e45cdd0cc06f8af383360b58109c030c233795 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Wed, 7 May 2025 15:02:08 +0530 Subject: [PATCH 3/8] Refactored ListContainers command to handle exclusive options using ArgGroup --- .../debug/logs/container/ListContainers.java | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java index 0d8819a8efaa..36601792bb17 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java @@ -35,21 +35,26 @@ description = "Finds containers from the database based on the option provided." ) public class ListContainers extends AbstractSubcommand implements Callable { - - @CommandLine.Option(names = {"--state"}, - description = "Life cycle state of the container.") - private HddsProtos.LifeCycleState state; + + @CommandLine.ArgGroup(multiplicity = "1") + private ExclusiveOptions exclusiveOptions; @CommandLine.Mixin private ListOptions listOptions; - @CommandLine.Option(names = {"--duplicate-open"}, - description = "List all the containers which have duplicate open states.") - private boolean duplicateOpen; - @CommandLine.ParentCommand private ContainerLogController parent; + private static final class ExclusiveOptions { + @CommandLine.Option(names = {"--state"}, + description = "Life cycle state of the container.") + private HddsProtos.LifeCycleState state; + + @CommandLine.Option(names = {"--duplicate-open"}, + description = "List all the containers which have duplicate open states.") + private boolean duplicateOpen; + } + @Override public Void call() throws Exception { @@ -60,12 +65,10 @@ public Void call() throws Exception { ContainerDatanodeDatabase cdd = new ContainerDatanodeDatabase(dbPath.toString()); - if (duplicateOpen) { + if (exclusiveOptions.duplicateOpen) { cdd.findDuplicateOpenContainer(); - } else if (state != null) { - cdd.listContainersByState(state.name(), listOptions.getLimit()); - } else { - err().println("Please provide either a container state or use --double-open."); + } else if (exclusiveOptions.state != null) { + cdd.listContainersByState(exclusiveOptions.state.name(), listOptions.getLimit()); } return null; From 28b09db151c0bc43f27379df4a2843dd006848b2 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Wed, 7 May 2025 20:04:16 +0530 Subject: [PATCH 4/8] Added occurrence for each container and updated query --- .../utils/ContainerDatanodeDatabase.java | 33 ++++++++++++++++--- .../logs/container/utils/SQLDBConstants.java | 3 ++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java index f72446ba19f5..f977b29a3112 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java @@ -562,12 +562,14 @@ public void findDuplicateOpenContainer() throws SQLException { while (resultSet.next()) { Long containerID = resultSet.getLong("container_id"); - List logEntries = getContainerLogData(containerID, connection); - logEntries.sort(Comparator.comparing(DatanodeContainerInfo::getTimestamp)); + List logEntries = getContainerLogDataFoOpen(containerID, connection); boolean hasIssue = checkForMultipleOpenStates(logEntries); if (hasIssue) { + int openStateCount = (int) logEntries.stream() + .filter(entry -> "OPEN".equalsIgnoreCase(entry.getState())) + .count(); count++; - out.println("Container ID: " + containerID); + out.println("Container ID: " + containerID + " - OPEN state count: " + openStateCount); } } @@ -575,10 +577,33 @@ public void findDuplicateOpenContainer() throws SQLException { } } catch (SQLException e) { - throw new SQLException("Error while retrieving containers."); + throw new SQLException("Error while retrieving containers." + e.getMessage(), e); } catch (Exception e) { throw new RuntimeException("Unexpected error: " + e); } } + + private List getContainerLogDataFoOpen(Long containerID, Connection connection) + throws SQLException { + String query = SQLDBConstants.SELECT_CONTAINER_DETAILS_OPEN_STATE; + List logEntries = new ArrayList<>(); + + try (PreparedStatement preparedStatement = connection.prepareStatement(query)) { + preparedStatement.setLong(1, containerID); + try (ResultSet rs = preparedStatement.executeQuery()) { + while (rs.next()) { + DatanodeContainerInfo entry = new DatanodeContainerInfo.Builder() + .setTimestamp(rs.getString("timestamp")) + .setContainerId(rs.getLong("container_id")) + .setDatanodeId(rs.getString("datanode_id")) + .setState(rs.getString("container_state")) + .build(); + logEntries.add(entry); + } + } + } + + return logEntries; + } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java index 67b5fa181aa9..74822d35e3d3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/SQLDBConstants.java @@ -72,6 +72,9 @@ public final class SQLDBConstants { "ON ContainerLogTable(container_id);"; public static final String SELECT_DISTINCT_CONTAINER_IDS_QUERY = "SELECT DISTINCT container_id FROM ContainerLogTable"; + public static final String SELECT_CONTAINER_DETAILS_OPEN_STATE = "SELECT d.timestamp, d.container_id, " + + "d.datanode_id, d.container_state FROM DatanodeContainerLogTable d " + + "WHERE d.container_id = ? AND d.container_state = 'OPEN' ORDER BY d.timestamp ASC;"; private SQLDBConstants() { //Never constructed From f4e81b25042dcbf43d501bc74782680f5989acae Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Wed, 7 May 2025 22:42:56 +0530 Subject: [PATCH 5/8] Fixed typo in method name --- .../debug/logs/container/utils/ContainerDatanodeDatabase.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java index f977b29a3112..22665d0616b5 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java @@ -562,7 +562,7 @@ public void findDuplicateOpenContainer() throws SQLException { while (resultSet.next()) { Long containerID = resultSet.getLong("container_id"); - List logEntries = getContainerLogDataFoOpen(containerID, connection); + List logEntries = getContainerLogDataForOpenContainers(containerID, connection); boolean hasIssue = checkForMultipleOpenStates(logEntries); if (hasIssue) { int openStateCount = (int) logEntries.stream() @@ -583,7 +583,7 @@ public void findDuplicateOpenContainer() throws SQLException { } } - private List getContainerLogDataFoOpen(Long containerID, Connection connection) + private List getContainerLogDataForOpenContainers(Long containerID, Connection connection) throws SQLException { String query = SQLDBConstants.SELECT_CONTAINER_DETAILS_OPEN_STATE; List logEntries = new ArrayList<>(); From d097cdb6d1f3251464e90795e4e70368af3d7cf6 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Thu, 8 May 2025 11:27:34 +0530 Subject: [PATCH 6/8] Updated index creation and made separate command to list duplicate open --- .../container/ContainerLogController.java | 1 + .../logs/container/ContainerLogParser.java | 1 + .../DuplicateOpenContainersCommand.java | 53 +++++++++++++++++++ .../debug/logs/container/ListContainers.java | 23 ++------ .../utils/ContainerDatanodeDatabase.java | 39 ++++++++------ 5 files changed, 82 insertions(+), 35 deletions(-) create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java index 1d4972b9c0a1..dd43119ea6ed 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogController.java @@ -33,6 +33,7 @@ subcommands = { ContainerInfoCommand.class, ContainerLogParser.class, + DuplicateOpenContainersCommand.class, ListContainers.class }, description = "Tool to parse and store container logs from datanodes into a temporary SQLite database." + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java index e2d472e2e95f..cac2518b381f 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ContainerLogParser.java @@ -89,6 +89,7 @@ public Void call() throws Exception { parser.processLogEntries(path, cdd, threadCount); cdd.insertLatestContainerLogData(); + cdd.createIndexes(); out().println("Successfully parsed the log files and updated the respective tables"); return null; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java new file mode 100644 index 000000000000..261ed57df059 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug.logs.container; + +import java.nio.file.Path; +import java.util.concurrent.Callable; +import org.apache.hadoop.hdds.cli.AbstractSubcommand; +import org.apache.hadoop.ozone.debug.logs.container.utils.ContainerDatanodeDatabase; +import picocli.CommandLine; + +/** + * Subcommand to list containers that have duplicate OPEN states. + */ + +@CommandLine.Command( + name = "duplicate-open", + description = "List all containers which have duplicate open states." + + "Outputs the container ID along with the count of OPEN state entries." +) +public class DuplicateOpenContainersCommand extends AbstractSubcommand implements Callable { + + @CommandLine.ParentCommand + private ContainerLogController parent; + + @Override + public Void call() throws Exception { + Path dbPath = parent.resolveDbPath(); + if (dbPath == null) { + return null; + } + + ContainerDatanodeDatabase cdd = new ContainerDatanodeDatabase(dbPath.toString()); + cdd.findDuplicateOpenContainer(); + + return null; + } +} + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java index 36601792bb17..bb32d26f2327 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java @@ -36,8 +36,10 @@ ) public class ListContainers extends AbstractSubcommand implements Callable { - @CommandLine.ArgGroup(multiplicity = "1") - private ExclusiveOptions exclusiveOptions; + @CommandLine.Option(names = {"--state"}, + description = "Life cycle state of the container.", + required = true) + private HddsProtos.LifeCycleState state; @CommandLine.Mixin private ListOptions listOptions; @@ -45,16 +47,6 @@ public class ListContainers extends AbstractSubcommand implements Callable @CommandLine.ParentCommand private ContainerLogController parent; - private static final class ExclusiveOptions { - @CommandLine.Option(names = {"--state"}, - description = "Life cycle state of the container.") - private HddsProtos.LifeCycleState state; - - @CommandLine.Option(names = {"--duplicate-open"}, - description = "List all the containers which have duplicate open states.") - private boolean duplicateOpen; - } - @Override public Void call() throws Exception { @@ -65,12 +57,7 @@ public Void call() throws Exception { ContainerDatanodeDatabase cdd = new ContainerDatanodeDatabase(dbPath.toString()); - if (exclusiveOptions.duplicateOpen) { - cdd.findDuplicateOpenContainer(); - } else if (exclusiveOptions.state != null) { - cdd.listContainersByState(exclusiveOptions.state.name(), listOptions.getLimit()); - } - + cdd.listContainersByState(state.name(), listOptions.getLimit()); return null; } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java index 22665d0616b5..6434852e90ed 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java @@ -116,6 +116,20 @@ private void createContainerLogTable() throws SQLException { } } + public void createIndexes() throws SQLException { + try (Connection connection = getConnection(); + Statement stmt = connection.createStatement()) { + createIdxDclContainerStateTime(stmt); + createContainerLogIndex(stmt); + createIdxContainerlogContainerId(stmt); + } catch (SQLException e) { + throw new SQLException("Error while creating index: " + e.getMessage()); + } catch (Exception e) { + throw new RuntimeException("Unexpected error: " + e); + } + } + + /** * Inserts a list of container log entries into the DatanodeContainerLogTable. * @@ -255,10 +269,7 @@ public void listContainersByState(String state, Integer limit) throws SQLExcepti String baseQuery = SQLDBConstants.SELECT_LATEST_CONTAINER_LOGS_BY_STATE; String finalQuery = limitProvided ? baseQuery + " LIMIT ?" : baseQuery; - try (Connection connection = getConnection(); - Statement stmt = connection.createStatement()) { - - createContainerLogIndex(stmt); + try (Connection connection = getConnection()) { try (PreparedStatement pstmt = connection.prepareStatement(finalQuery)) { pstmt.setString(1, state); @@ -304,11 +315,9 @@ public void listContainersByState(String state, Integer limit) throws SQLExcepti } } - private void createIdxDclContainerStateTime(Connection conn) throws SQLException { - String sql = SQLDBConstants.CREATE_DCL_CONTAINER_STATE_TIME_INDEX; - try (Statement stmt = conn.createStatement()) { - stmt.execute(sql); - } + private void createIdxDclContainerStateTime(Statement stmt) throws SQLException { + String createIndexSQL = SQLDBConstants.CREATE_DCL_CONTAINER_STATE_TIME_INDEX; + stmt.execute(createIndexSQL); } /** @@ -323,7 +332,7 @@ private void createIdxDclContainerStateTime(Connection conn) throws SQLException public void showContainerDetails(Long containerID) throws SQLException { try (Connection connection = getConnection()) { - createIdxDclContainerStateTime(connection); + List logEntries = getContainerLogData(containerID, connection); if (logEntries.isEmpty()) { @@ -542,11 +551,9 @@ private List getContainerLogData(Long containerID, Connec return logEntries; } - private void createIdxContainerlogContainerId(Connection conn) throws SQLException { - String sql = SQLDBConstants.CREATE_CONTAINER_ID_INDEX; - try (Statement stmt = conn.createStatement()) { - stmt.execute(sql); - } + private void createIdxContainerlogContainerId(Statement stmt) throws SQLException { + String createIndexSQL = SQLDBConstants.CREATE_CONTAINER_ID_INDEX; + stmt.execute(createIndexSQL); } public void findDuplicateOpenContainer() throws SQLException { @@ -554,8 +561,6 @@ public void findDuplicateOpenContainer() throws SQLException { try (Connection connection = getConnection()) { - createIdxContainerlogContainerId(connection); - try (PreparedStatement statement = connection.prepareStatement(sql); ResultSet resultSet = statement.executeQuery()) { int count = 0; From 87b998edc1d196e6e4c69c7213e5c8d6b3f034d8 Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Thu, 8 May 2025 11:37:42 +0530 Subject: [PATCH 7/8] Fixed checkstyle --- .../debug/logs/container/utils/ContainerDatanodeDatabase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java index 6434852e90ed..93455d67ee0b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java @@ -129,7 +129,6 @@ public void createIndexes() throws SQLException { } } - /** * Inserts a list of container log entries into the DatanodeContainerLogTable. * From 6670922adae0e3c32f005ab6f28b4fa7a7ed759d Mon Sep 17 00:00:00 2001 From: Sreeja Chintalapati Date: Thu, 8 May 2025 12:56:33 +0530 Subject: [PATCH 8/8] Removed unused AbstractSubcommand and reorganized index methods --- .../DuplicateOpenContainersCommand.java | 3 +- .../debug/logs/container/ListContainers.java | 6 ++-- .../utils/ContainerDatanodeDatabase.java | 30 +++++++++---------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java index 261ed57df059..820d5b5d2e41 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/DuplicateOpenContainersCommand.java @@ -19,7 +19,6 @@ import java.nio.file.Path; import java.util.concurrent.Callable; -import org.apache.hadoop.hdds.cli.AbstractSubcommand; import org.apache.hadoop.ozone.debug.logs.container.utils.ContainerDatanodeDatabase; import picocli.CommandLine; @@ -32,7 +31,7 @@ description = "List all containers which have duplicate open states." + "Outputs the container ID along with the count of OPEN state entries." ) -public class DuplicateOpenContainersCommand extends AbstractSubcommand implements Callable { +public class DuplicateOpenContainersCommand implements Callable { @CommandLine.ParentCommand private ContainerLogController parent; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java index bb32d26f2327..25a5e7c003e3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/ListContainers.java @@ -19,7 +19,6 @@ import java.nio.file.Path; import java.util.concurrent.Callable; -import org.apache.hadoop.hdds.cli.AbstractSubcommand; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.debug.logs.container.utils.ContainerDatanodeDatabase; import org.apache.hadoop.ozone.shell.ListOptions; @@ -34,8 +33,8 @@ name = "list", description = "Finds containers from the database based on the option provided." ) -public class ListContainers extends AbstractSubcommand implements Callable { - +public class ListContainers implements Callable { + @CommandLine.Option(names = {"--state"}, description = "Life cycle state of the container.", required = true) @@ -58,6 +57,7 @@ public Void call() throws Exception { ContainerDatanodeDatabase cdd = new ContainerDatanodeDatabase(dbPath.toString()); cdd.listContainersByState(state.name(), listOptions.getLimit()); + return null; } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java index 93455d67ee0b..1d44605249c8 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/logs/container/utils/ContainerDatanodeDatabase.java @@ -129,6 +129,21 @@ public void createIndexes() throws SQLException { } } + private void createIdxDclContainerStateTime(Statement stmt) throws SQLException { + String createIndexSQL = SQLDBConstants.CREATE_DCL_CONTAINER_STATE_TIME_INDEX; + stmt.execute(createIndexSQL); + } + + private void createContainerLogIndex(Statement stmt) throws SQLException { + String createIndexSQL = SQLDBConstants.CREATE_INDEX_LATEST_STATE; + stmt.execute(createIndexSQL); + } + + private void createIdxContainerlogContainerId(Statement stmt) throws SQLException { + String createIndexSQL = SQLDBConstants.CREATE_CONTAINER_ID_INDEX; + stmt.execute(createIndexSQL); + } + /** * Inserts a list of container log entries into the DatanodeContainerLogTable. * @@ -241,11 +256,6 @@ private void dropTable(String tableName, Statement stmt) throws SQLException { stmt.executeUpdate(dropTableSQL); } - private void createContainerLogIndex(Statement stmt) throws SQLException { - String createIndexSQL = SQLDBConstants.CREATE_INDEX_LATEST_STATE; - stmt.execute(createIndexSQL); - } - /** * Lists containers filtered by the specified state and writes their details to stdout * unless redirected to a file explicitly. @@ -314,11 +324,6 @@ public void listContainersByState(String state, Integer limit) throws SQLExcepti } } - private void createIdxDclContainerStateTime(Statement stmt) throws SQLException { - String createIndexSQL = SQLDBConstants.CREATE_DCL_CONTAINER_STATE_TIME_INDEX; - stmt.execute(createIndexSQL); - } - /** * Displays detailed information about a container based on its ID, including its state, BCSID, * timestamp, message, and index value. It also checks for issues such as UNHEALTHY @@ -550,11 +555,6 @@ private List getContainerLogData(Long containerID, Connec return logEntries; } - private void createIdxContainerlogContainerId(Statement stmt) throws SQLException { - String createIndexSQL = SQLDBConstants.CREATE_CONTAINER_ID_INDEX; - stmt.execute(createIndexSQL); - } - public void findDuplicateOpenContainer() throws SQLException { String sql = SQLDBConstants.SELECT_DISTINCT_CONTAINER_IDS_QUERY;