From 52f8721c4096706f4d41ac86197aae6d9366e08e Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 8 Apr 2025 14:22:36 +0530 Subject: [PATCH 01/13] HDDS-12207. Unify output of checks --- .../replicas/BlockExistenceVerifier.java | 86 +++++++++++ .../replicas/BlockVerificationResult.java | 91 ++++++++++++ .../ozone/debug/replicas/Checksums.java | 134 ++--------------- .../ozone/debug/replicas/ReplicaVerifier.java | 8 +- .../ozone/debug/replicas/ReplicasVerify.java | 135 ++++++++++++++++-- 5 files changed, 317 insertions(+), 137 deletions(-) create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java new file mode 100644 index 000000000000..aee959edd6a6 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug.replicas; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.XceiverClientManager; +import org.apache.hadoop.hdds.scm.XceiverClientSpi; +import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; + +/** + * Checks block existence using GetBlock calls to the Datanodes. + */ +public class BlockExistenceVerifier implements ReplicaVerifier { + + private OzoneConfiguration conf; + private final String checkType = "blockExistence"; + + public BlockExistenceVerifier(OzoneConfiguration conf) { + this.conf = conf; + } + + @Override + public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OzoneInputStream stream, + OmKeyLocationInfo keyLocation) { + try (ContainerOperationClient containerClient = new ContainerOperationClient(conf); + XceiverClientManager xceiverClientManager = containerClient.getXceiverClientManager()) { + + Pipeline keyPipeline = keyLocation.getPipeline(); + boolean isECKey = keyPipeline.getReplicationConfig().getReplicationType() == HddsProtos.ReplicationType.EC; + Pipeline pipeline = isECKey ? keyPipeline : + Pipeline.newBuilder(keyPipeline) + .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)) + .build(); + + try (XceiverClientSpi client = xceiverClientManager.acquireClientForReadData(pipeline)) { + + Map responses = + ContainerProtocolCalls.getBlockFromAllNodes( + client, + keyLocation.getBlockID().getDatanodeBlockIDProtobuf(), + keyLocation.getToken() + ); + + ContainerProtos.GetBlockResponseProto response = responses.get(datanode); + boolean hasBlock = response != null && response.hasBlockData(); + + return new BlockVerificationResult(checkType, hasBlock, hasBlock ? Collections.emptyList() : + Collections.singletonList(new BlockVerificationResult.FailureDetail( + true, "Block does not exist on this replica"))); + } + + } catch (IOException | InterruptedException e) { + BlockVerificationResult.FailureDetail failure = new BlockVerificationResult.FailureDetail(true, e.getMessage()); + return new BlockVerificationResult(checkType, false, Collections.singletonList(failure)); + } + } + +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java new file mode 100644 index 000000000000..873774306f32 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug.replicas; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.List; + +/** + * Json structure for replicas to pass through each check and give output. + */ +public class BlockVerificationResult { + + private final String type; + private final boolean pass; + private final List failures; + + public BlockVerificationResult(String type, boolean pass, List failures) { + this.type = type; + this.pass = pass; + this.failures = failures; + } + + public String getType() { + return type; + } + + public boolean isPass() { + return pass; + } + + public List getFailures() { + return failures; + } + + /** + * Details about the check failure. + */ + public static class FailureDetail { + private final boolean present; + private final String message; + + public FailureDetail(boolean present, String message) { + this.present = present; + this.message = message; + } + + public boolean isPresent() { + return present; + } + + public String getFailureMessage() { + return message; + } + + } + + public ObjectNode toJson(ObjectMapper mapper) { + ObjectNode resultNode = mapper.createObjectNode(); + resultNode.put("type", type); + resultNode.put("pass", pass); + + ArrayNode failuresArray = mapper.createArrayNode(); + for (FailureDetail failure : failures) { + ObjectNode failureNode = mapper.createObjectNode(); + failureNode.put("present", failure.isPresent()); + failureNode.put("message", failure.getFailureMessage()); + failuresArray.add(failureNode); + } + + resultNode.set("failures", failuresArray); + return resultNode; + } + +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java index 38f6f0103017..90cc15d7a503 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java @@ -17,25 +17,13 @@ package org.apache.hadoop.ozone.debug.replicas; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import jakarta.annotation.Nonnull; -import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; +import java.util.Collections; import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.NullOutputStream; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.server.JsonUtils; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; /** @@ -44,121 +32,23 @@ * downloaded replicas. */ public class Checksums implements ReplicaVerifier { + private final String checkType = "checksum"; - private static final String JSON_PROPERTY_FILE_NAME = "filename"; - private static final String JSON_PROPERTY_FILE_SIZE = "datasize"; - private static final String JSON_PROPERTY_FILE_BLOCKS = "blocks"; - private static final String JSON_PROPERTY_BLOCK_INDEX = "blockIndex"; - private static final String JSON_PROPERTY_BLOCK_CONTAINERID = "containerId"; - private static final String JSON_PROPERTY_BLOCK_LOCALID = "localId"; - private static final String JSON_PROPERTY_BLOCK_LENGTH = "length"; - private static final String JSON_PROPERTY_BLOCK_OFFSET = "offset"; - private static final String JSON_PROPERTY_BLOCK_REPLICAS = "replicas"; - private static final String JSON_PROPERTY_REPLICA_HOSTNAME = "hostname"; - private static final String JSON_PROPERTY_REPLICA_UUID = "uuid"; - private static final String JSON_PROPERTY_REPLICA_EXCEPTION = "exception"; - - - private String outputDir; - private OzoneClient client; - - public Checksums(OzoneClient client, String outputDir) { - this.client = client; - this.outputDir = outputDir; - } - - private void downloadReplicasAndCreateManifest( - Map> replicas, - ArrayNode blocks) throws IOException { - int blockIndex = 0; - - for (Map.Entry> - block : replicas.entrySet()) { - ObjectNode blockJson = JsonUtils.createObjectNode(null); - ArrayNode replicasJson = JsonUtils.createArrayNode(); - - blockIndex += 1; - OmKeyLocationInfo locationInfo = block.getKey(); - blockJson.put(JSON_PROPERTY_BLOCK_INDEX, blockIndex); - blockJson.put(JSON_PROPERTY_BLOCK_CONTAINERID, locationInfo.getContainerID()); - blockJson.put(JSON_PROPERTY_BLOCK_LOCALID, locationInfo.getLocalID()); - blockJson.put(JSON_PROPERTY_BLOCK_LENGTH, locationInfo.getLength()); - blockJson.put(JSON_PROPERTY_BLOCK_OFFSET, locationInfo.getOffset()); - - for (Map.Entry - replica : block.getValue().entrySet()) { - DatanodeDetails datanode = replica.getKey(); - - ObjectNode replicaJson = JsonUtils.createObjectNode(null); - - replicaJson.put(JSON_PROPERTY_REPLICA_HOSTNAME, datanode.getHostName()); - replicaJson.put(JSON_PROPERTY_REPLICA_UUID, datanode.getUuidString()); - - try (InputStream is = replica.getValue()) { - IOUtils.copyLarge(is, NullOutputStream.INSTANCE); - } catch (IOException e) { - replicaJson.put(JSON_PROPERTY_REPLICA_EXCEPTION, e.getMessage()); - } - replicasJson.add(replicaJson); - } - blockJson.set(JSON_PROPERTY_BLOCK_REPLICAS, replicasJson); - blocks.add(blockJson); - } - } - - @Nonnull - private File createDirectory(String volumeName, String bucketName, - String keyName) throws IOException { - String fileSuffix - = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); - String directoryName = volumeName + "_" + bucketName + "_" + keyName + - "_" + fileSuffix; - System.out.println("Creating directory : " + directoryName); - File dir = new File(outputDir, directoryName); - if (!dir.exists()) { - if (dir.mkdirs()) { - System.out.println("Successfully created!"); - } else { - throw new IOException(String.format( - "Failed to create directory %s.", dir)); - } - } - return dir; + public Checksums() { } @Override - public void verifyKey(OzoneKeyDetails keyDetails) { - String volumeName = keyDetails.getVolumeName(); - String bucketName = keyDetails.getBucketName(); - String keyName = keyDetails.getName(); - System.out.println("Processing key : " + volumeName + "/" + bucketName + "/" + keyName); - try { - ClientProtocol checksumClient = client.getObjectStore().getClientProxy(); - - // Multilevel keys will have a '/' in their names. This interferes with - // directory and file creation process. Flatten the keys to fix this. - String sanitizedKeyName = keyName.replace("/", "_"); + public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OzoneInputStream stream, + OmKeyLocationInfo keyLocation) { + try (InputStream is = stream) { + IOUtils.copyLarge(is, NullOutputStream.INSTANCE); - File dir = createDirectory(volumeName, bucketName, sanitizedKeyName); - OzoneKeyDetails keyInfoDetails = checksumClient.getKeyDetails(volumeName, bucketName, keyName); - Map> replicas = - checksumClient.getKeysEveryReplicas(volumeName, bucketName, keyName); - - ObjectNode result = JsonUtils.createObjectNode(null); - result.put(JSON_PROPERTY_FILE_NAME, volumeName + "/" + bucketName + "/" + keyName); - result.put(JSON_PROPERTY_FILE_SIZE, keyInfoDetails.getDataSize()); - - ArrayNode blocks = JsonUtils.createArrayNode(); - downloadReplicasAndCreateManifest(replicas, blocks); - result.set(JSON_PROPERTY_FILE_BLOCKS, blocks); - - String prettyJson = JsonUtils.toJsonStringWithDefaultPrettyPrinter(result); - - String manifestFileName = sanitizedKeyName + "_manifest"; - File manifestFile = new File(dir, manifestFileName); - Files.write(manifestFile.toPath(), prettyJson.getBytes(StandardCharsets.UTF_8)); + return new BlockVerificationResult(checkType, true, Collections.emptyList()); } catch (IOException e) { - throw new RuntimeException(e); + BlockVerificationResult.FailureDetail failure = new BlockVerificationResult + .FailureDetail(true, e.getMessage()); + return new BlockVerificationResult(checkType, false, Collections.singletonList(failure)); } } + } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java index cbb5f31c981b..8472c5fb07f0 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java @@ -17,12 +17,14 @@ package org.apache.hadoop.ozone.debug.replicas; -import org.apache.hadoop.ozone.client.OzoneKeyDetails; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; /** - * Functional interface for implementing a key verifier. + * Functional interface for implementing a block verifier. */ @FunctionalInterface public interface ReplicaVerifier { - void verifyKey(OzoneKeyDetails keyDetails); + BlockVerificationResult verifyBlock(DatanodeDetails datanode, OzoneInputStream stream, OmKeyLocationInfo keyLocation); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java index 752b5f99e9b7..5993b51db2dc 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java @@ -17,11 +17,17 @@ package org.apache.hadoop.ozone.debug.replicas; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Map; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.cli.ScmOption; +import org.apache.hadoop.hdds.server.JsonUtils; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; @@ -29,6 +35,9 @@ import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.client.protocol.ClientProtocol; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.shell.Handler; import org.apache.hadoop.ozone.shell.OzoneAddress; import org.apache.hadoop.ozone.shell.Shell; @@ -40,7 +49,7 @@ @CommandLine.Command( name = "verify", - description = "Run checks to verify data across replicas") + description = "Run checks to verify data across replicas. By default prints only the keys with failed checks.") public class ReplicasVerify extends Handler { @CommandLine.Mixin private ScmOption scmOption; @@ -54,6 +63,10 @@ public class ReplicasVerify extends Handler { required = true) private String outputDir; + @CommandLine.Option(names = {"--all"}, + description = "Print results for all passing and failing keys") + private boolean allKeys; + @CommandLine.ArgGroup(exclusive = false, multiplicity = "1") private Verification verification; @@ -64,6 +77,11 @@ static class Verification { defaultValue = "false") private boolean doExecuteChecksums; + @CommandLine.Option(names = "--block-existence", + description = "Check for block existence on datanodes.", + defaultValue = "false") + private boolean doExecuteBlockExistence; + } private List replicaVerifiers; @@ -72,7 +90,11 @@ protected void execute(OzoneClient client, OzoneAddress address) throws IOExcept replicaVerifiers = new ArrayList<>(); if (verification.doExecuteChecksums) { - replicaVerifiers.add(new Checksums(client, outputDir)); + replicaVerifiers.add(new Checksums()); + } + + if (verification.doExecuteBlockExistence) { + replicaVerifiers.add(new BlockExistenceVerifier(getConf())); } findCandidateKeys(client, address); @@ -88,41 +110,130 @@ void findCandidateKeys(OzoneClient ozoneClient, OzoneAddress address) throws IOE String volumeName = address.getVolumeName(); String bucketName = address.getBucketName(); String keyName = address.getKeyName(); + + ObjectMapper mapper = new ObjectMapper(); + ObjectNode root = mapper.createObjectNode(); + ArrayNode keysArray = mapper.createArrayNode(); + if (!keyName.isEmpty()) { OzoneKeyDetails keyDetails = ozoneClient.getProxy().getKeyDetails(volumeName, bucketName, keyName); - processKey(keyDetails); + processKey(ozoneClient, keyDetails, keysArray); } else if (!bucketName.isEmpty()) { OzoneVolume volume = objectStore.getVolume(volumeName); OzoneBucket bucket = volume.getBucket(bucketName); - checkBucket(bucket); + checkBucket(ozoneClient, bucket, keysArray); } else if (!volumeName.isEmpty()) { OzoneVolume volume = objectStore.getVolume(volumeName); - checkVolume(volume); + checkVolume(ozoneClient, volume, keysArray); } else { for (Iterator it = objectStore.listVolumes(null); it.hasNext();) { - checkVolume(it.next()); + checkVolume(ozoneClient, it.next(), keysArray); } } + + root.set("keys", keysArray); + System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(root)); } - void checkVolume(OzoneVolume volume) throws IOException { + void checkVolume(OzoneClient ozoneClient, OzoneVolume volume, ArrayNode keysArray) throws IOException { for (Iterator it = volume.listBuckets(null); it.hasNext();) { OzoneBucket bucket = it.next(); - checkBucket(bucket); + checkBucket(ozoneClient, bucket, keysArray); } } - void checkBucket(OzoneBucket bucket) throws IOException { + void checkBucket(OzoneClient ozoneClient, OzoneBucket bucket, ArrayNode keysArray) throws IOException { for (Iterator it = bucket.listKeys(null); it.hasNext();) { OzoneKey key = it.next(); // TODO: Remove this check once HDDS-12094 is fixed if (!key.getName().endsWith("/")) { - processKey(bucket.getKey(key.getName())); + processKey(ozoneClient, bucket.getKey(key.getName()), keysArray); } } } - void processKey(OzoneKeyDetails keyDetails) { - replicaVerifiers.forEach(verifier -> verifier.verifyKey(keyDetails)); + void processKey(OzoneClient ozoneClient, OzoneKeyDetails keyDetails, ArrayNode keysArray) { + ObjectMapper mapper = new ObjectMapper(); + + String volumeName = keyDetails.getVolumeName(); + String bucketName = keyDetails.getBucketName(); + String keyName = keyDetails.getName(); + + ObjectNode keyNode = JsonUtils.createObjectNode(null); + keyNode.put("volumeName", volumeName); + keyNode.put("bucketName", bucketName); + keyNode.put("name", keyName); + + ArrayNode blocksArray = mapper.createArrayNode(); + boolean keyPass = true; //to check if all checks are passed + + try { + ClientProtocol clientProtocol = ozoneClient.getObjectStore().getClientProxy(); + Map> replicas = + clientProtocol.getKeysEveryReplicas(volumeName, bucketName, keyName); + + for (Map.Entry> block : replicas.entrySet()) { + OmKeyLocationInfo keyLocation = block.getKey(); + long containerID = keyLocation.getContainerID(); + long localID = keyLocation.getLocalID(); + + ObjectNode blockNode = JsonUtils.createObjectNode(null); + blockNode.put("containerID", containerID); + blockNode.put("localID", localID); + + ArrayNode replicasArray = mapper.createArrayNode(); + boolean blockPass = !block.getValue().isEmpty(); // If no replicas, the block automatically fails + + for (Map.Entry replica : block.getValue().entrySet()) { + DatanodeDetails datanode = replica.getKey(); + + ObjectNode datanodeNode = JsonUtils.createObjectNode(null); + datanodeNode.put("uuid", datanode.getUuidString()); + datanodeNode.put("hostname", datanode.getHostName()); + + + ArrayNode checksArray = mapper.createArrayNode(); + boolean replicaPass = true; + + ObjectMapper replicaVerifierNode = new ObjectMapper(); + for (ReplicaVerifier verifier : replicaVerifiers) { + BlockVerificationResult result = verifier.verifyBlock(replica.getKey(), replica.getValue(), keyLocation); + ObjectNode checksNode = result.toJson(replicaVerifierNode); + checksArray.add(checksNode); + + if (!result.isPass()) { + replicaPass = false; + } + } + + ObjectNode replicaNode = mapper.createObjectNode(); + replicaNode.set("datanode", datanodeNode); + replicaNode.set("checks", checksArray); + //replicaNode.put("pass", replicaPass); // ← include pass flag for replica if needed + + replicasArray.add(replicaNode); + + if (!replicaPass) { + blockPass = false; + } + } + blockNode.set("replicas", replicasArray); + // blockNode.put("pass", blockPass); // ← pass flag per block + + blocksArray.add(blockNode); + + if (!blockPass) { + keyPass = false; + } + } + } catch (IOException e) { + throw new RuntimeException("Error retrieving replicas for key: " + keyName); + } + + if (!keyPass | allKeys) { + keyNode.set("blocks", blocksArray); + keyNode.put("pass", keyPass); + keysArray.add(keyNode); + } } } From c61755558fc13273c28e1498c75a6a926a33dddb Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 8 Apr 2025 15:35:01 +0530 Subject: [PATCH 02/13] Findbugs addressed --- .../hadoop/ozone/debug/replicas/BlockExistenceVerifier.java | 6 +++--- .../org/apache/hadoop/ozone/debug/replicas/Checksums.java | 6 +++--- .../apache/hadoop/ozone/debug/replicas/ReplicasVerify.java | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java index aee959edd6a6..9a5338838bdd 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java @@ -41,7 +41,7 @@ public class BlockExistenceVerifier implements ReplicaVerifier { private OzoneConfiguration conf; - private final String checkType = "blockExistence"; + private static final String CHECKTYPE = "blockExistence"; public BlockExistenceVerifier(OzoneConfiguration conf) { this.conf = conf; @@ -72,14 +72,14 @@ public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OzoneInputS ContainerProtos.GetBlockResponseProto response = responses.get(datanode); boolean hasBlock = response != null && response.hasBlockData(); - return new BlockVerificationResult(checkType, hasBlock, hasBlock ? Collections.emptyList() : + return new BlockVerificationResult(CHECKTYPE, hasBlock, hasBlock ? Collections.emptyList() : Collections.singletonList(new BlockVerificationResult.FailureDetail( true, "Block does not exist on this replica"))); } } catch (IOException | InterruptedException e) { BlockVerificationResult.FailureDetail failure = new BlockVerificationResult.FailureDetail(true, e.getMessage()); - return new BlockVerificationResult(checkType, false, Collections.singletonList(failure)); + return new BlockVerificationResult(CHECKTYPE, false, Collections.singletonList(failure)); } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java index 90cc15d7a503..1011a9f7e138 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java @@ -32,7 +32,7 @@ * downloaded replicas. */ public class Checksums implements ReplicaVerifier { - private final String checkType = "checksum"; + private static final String CHECKTYPE = "checksum"; public Checksums() { } @@ -43,11 +43,11 @@ public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OzoneInputS try (InputStream is = stream) { IOUtils.copyLarge(is, NullOutputStream.INSTANCE); - return new BlockVerificationResult(checkType, true, Collections.emptyList()); + return new BlockVerificationResult(CHECKTYPE, true, Collections.emptyList()); } catch (IOException e) { BlockVerificationResult.FailureDetail failure = new BlockVerificationResult .FailureDetail(true, e.getMessage()); - return new BlockVerificationResult(checkType, false, Collections.singletonList(failure)); + return new BlockVerificationResult(CHECKTYPE, false, Collections.singletonList(failure)); } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java index 5993b51db2dc..b991f7d24547 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java @@ -230,7 +230,7 @@ void processKey(OzoneClient ozoneClient, OzoneKeyDetails keyDetails, ArrayNode k throw new RuntimeException("Error retrieving replicas for key: " + keyName); } - if (!keyPass | allKeys) { + if (!keyPass || allKeys) { keyNode.set("blocks", blocksArray); keyNode.put("pass", keyPass); keysArray.add(keyNode); From 726af03fc8ed3a7aa0a74a93bdffe8d98b7924d5 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Mon, 21 Apr 2025 21:44:30 +0530 Subject: [PATCH 03/13] Modified the verifiers --- .../hadoop/ozone/client/rpc/RpcClient.java | 2 +- .../src/main/compose/common/replicas-test.sh | 50 +++++----- .../smoketest/debug/ozone-debug-tests.robot | 10 +- .../replicas/BlockExistenceVerifier.java | 68 +++++++------ .../replicas/BlockVerificationResult.java | 58 +++++------ .../debug/replicas/ChecksumVerifier.java | 91 ++++++++++++++++++ .../ozone/debug/replicas/Checksums.java | 54 ----------- .../ozone/debug/replicas/ReplicaVerifier.java | 6 +- .../ozone/debug/replicas/ReplicasVerify.java | 95 +++++++++---------- 9 files changed, 225 insertions(+), 209 deletions(-) create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 4c0e99a44693..11d96393c04c 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1850,7 +1850,7 @@ private OmKeyInfo getS3PartKeyInfo( return keyInfoWithS3Context.getKeyInfo(); } - private OmKeyInfo getKeyInfo( + public OmKeyInfo getKeyInfo( String volumeName, String bucketName, String keyName, boolean forceUpdateContainerCache) throws IOException { Preconditions.checkNotNull(volumeName); diff --git a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh index cd129bb07872..485e189eed5b 100755 --- a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh @@ -24,29 +24,29 @@ key="testfile" execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests.robot # get block locations for key -chunkinfo="${key}-blocks-${prefix}" -docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info ${volume}/${bucket}/${key}" > "$chunkinfo" -host="$(jq -r '.KeyLocations[0][0]["Datanode-HostName"]' ${chunkinfo})" -container="${host%%.*}" - -# corrupt the first block of key on one of the datanodes -datafile="$(jq -r '.KeyLocations[0][0].Locations.files[0]' ${chunkinfo})" -docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}" - -execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}" debug/ozone-debug-corrupt-block.robot - -docker stop "${container}" - -wait_for_datanode "${container}" STALE 60 -execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}" debug/ozone-debug-stale-datanode.robot - -wait_for_datanode "${container}" DEAD 60 -execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-dead-datanode.robot - -docker start "${container}" - -wait_for_datanode "${container}" HEALTHY 60 +#chunkinfo="${key}-blocks-${prefix}" +#docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info ${volume}/${bucket}/${key}" > "$chunkinfo" +#host="$(jq -r '.KeyLocations[0][0]["Datanode-HostName"]' ${chunkinfo})" +#container="${host%%.*}" +# +## corrupt the first block of key on one of the datanodes +#datafile="$(jq -r '.KeyLocations[0][0].Locations.files[0]' ${chunkinfo})" +#docker exec "${container}" sed -i -e '1s/^/a/' "${datafile}" +# +#execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "CORRUPT_DATANODE:${host}" debug/ozone-debug-corrupt-block.robot +# +#docker stop "${container}" +# +#wait_for_datanode "${container}" STALE 60 +#execute_robot_test ${SCM} -v "PREFIX:${prefix}" -v "STALE_DATANODE:${host}" debug/ozone-debug-stale-datanode.robot +# +#wait_for_datanode "${container}" DEAD 60 +#execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-dead-datanode.robot +# +#docker start "${container}" +# +#wait_for_datanode "${container}" HEALTHY 60 -start_docker_env 9 -execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot -execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec6-3.robot \ No newline at end of file +#start_docker_env 9 +#execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec3-2.robot +#execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests-ec6-3.robot diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot index 02de6794b237..d64d0c0c9744 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot @@ -36,12 +36,12 @@ Write keys Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/${TESTFILE} ${TEMP_DIR}/${TESTFILE} *** Test Cases *** -Test ozone debug read-replicas - ${directory} = Execute replicas verify checksums CLI tool - Set Test Variable ${DIR} ${directory} +#Test ozone debug read-replicas +# ${directory} = Execute replicas verify checksums CLI tool +# Set Test Variable ${DIR} ${directory} - ${count_files} = Count Files In Directory ${directory} - Should Be Equal As Integers ${count_files} 1 +# ${count_files} = Count Files In Directory ${directory} +# Should Be Equal As Integers ${count_files} 1 Test ozone debug version diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java index 9a5338838bdd..c34950b62356 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java @@ -21,66 +21,62 @@ import java.io.IOException; import java.util.Collections; -import java.util.Map; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; /** - * Checks block existence using GetBlock calls to the Datanodes. + * Verifies block existence by making getBlock calls to the datanode. */ public class BlockExistenceVerifier implements ReplicaVerifier { + private final OzoneConfiguration conf; + private final ContainerOperationClient containerClient; + private final XceiverClientManager xceiverClientManager; + private static final String CHECK_TYPE = "blockExistence"; - private OzoneConfiguration conf; - private static final String CHECKTYPE = "blockExistence"; + @Override + public String getType() { + return CHECK_TYPE; + } - public BlockExistenceVerifier(OzoneConfiguration conf) { + public BlockExistenceVerifier(OzoneConfiguration conf) throws IOException { this.conf = conf; + this.containerClient = new ContainerOperationClient(conf); + this.xceiverClientManager = containerClient.getXceiverClientManager(); } @Override - public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OzoneInputStream stream, - OmKeyLocationInfo keyLocation) { - try (ContainerOperationClient containerClient = new ContainerOperationClient(conf); - XceiverClientManager xceiverClientManager = containerClient.getXceiverClientManager()) { - - Pipeline keyPipeline = keyLocation.getPipeline(); - boolean isECKey = keyPipeline.getReplicationConfig().getReplicationType() == HddsProtos.ReplicationType.EC; - Pipeline pipeline = isECKey ? keyPipeline : - Pipeline.newBuilder(keyPipeline) - .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)) - .build(); + public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocationInfo keyLocation) { + try { + Pipeline pipeline = Pipeline.newBuilder(keyLocation.getPipeline()) + .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)) + .setNodes(Collections.singletonList(datanode)) + .build(); - try (XceiverClientSpi client = xceiverClientManager.acquireClientForReadData(pipeline)) { + XceiverClientSpi client = xceiverClientManager.acquireClientForReadData(pipeline); + ContainerProtos.GetBlockResponseProto response = ContainerProtocolCalls.getBlock( + client, + keyLocation.getBlockID(), + keyLocation.getToken(), + Collections.singletonMap(datanode, 1) + ); - Map responses = - ContainerProtocolCalls.getBlockFromAllNodes( - client, - keyLocation.getBlockID().getDatanodeBlockIDProtobuf(), - keyLocation.getToken() - ); + boolean hasBlock = response != null && response.hasBlockData(); - ContainerProtos.GetBlockResponseProto response = responses.get(datanode); - boolean hasBlock = response != null && response.hasBlockData(); - - return new BlockVerificationResult(CHECKTYPE, hasBlock, hasBlock ? Collections.emptyList() : - Collections.singletonList(new BlockVerificationResult.FailureDetail( - true, "Block does not exist on this replica"))); + if (hasBlock) { + return BlockVerificationResult.pass(); + } else { + return BlockVerificationResult.failCheck("Block does not exist on this replica"); } - - } catch (IOException | InterruptedException e) { - BlockVerificationResult.FailureDetail failure = new BlockVerificationResult.FailureDetail(true, e.getMessage()); - return new BlockVerificationResult(CHECKTYPE, false, Collections.singletonList(failure)); + } catch (IOException e) { + return BlockVerificationResult.failIncomplete(e.getMessage()); } } - } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java index 873774306f32..50702d84fadf 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java @@ -17,52 +17,61 @@ package org.apache.hadoop.ozone.debug.replicas; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.Collections; import java.util.List; +import java.util.Optional; /** * Json structure for replicas to pass through each check and give output. */ public class BlockVerificationResult { - private final String type; private final boolean pass; private final List failures; - public BlockVerificationResult(String type, boolean pass, List failures) { - this.type = type; + public BlockVerificationResult(boolean pass, List failures) { this.pass = pass; this.failures = failures; } - public String getType() { - return type; + public static BlockVerificationResult pass() { + return new BlockVerificationResult(true, null); } - public boolean isPass() { + public static BlockVerificationResult failCheck(String message) { + return new BlockVerificationResult(false, + Collections.singletonList(new FailureDetail(true, message))); + } + + public static BlockVerificationResult failIncomplete(String message) { + return new BlockVerificationResult(false, + Collections.singletonList(new FailureDetail(false, message))); + } + + public boolean passed() { return pass; } - public List getFailures() { - return failures; + public Optional> getFailures() { + return Optional.ofNullable(failures); } /** * Details about the check failure. */ public static class FailureDetail { - private final boolean present; + // indicates whether the check finished and failed, + // or it was unable to finish due to connection or other issues + private final boolean completed; private final String message; - public FailureDetail(boolean present, String message) { - this.present = present; + public FailureDetail(boolean completed, String message) { + this.completed = completed; this.message = message; } - public boolean isPresent() { - return present; + public boolean isCompleted() { + return completed; } public String getFailureMessage() { @@ -71,21 +80,4 @@ public String getFailureMessage() { } - public ObjectNode toJson(ObjectMapper mapper) { - ObjectNode resultNode = mapper.createObjectNode(); - resultNode.put("type", type); - resultNode.put("pass", pass); - - ArrayNode failuresArray = mapper.createArrayNode(); - for (FailureDetail failure : failures) { - ObjectNode failureNode = mapper.createObjectNode(); - failureNode.put("present", failure.isPresent()); - failureNode.put("message", failure.getFailureMessage()); - failuresArray.add(failureNode); - } - - resultNode.set("failures", failuresArray); - return resultNode; - } - } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java new file mode 100644 index 000000000000..df25085a8818 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.debug.replicas; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.NullOutputStream; +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; +import org.apache.hadoop.hdds.scm.XceiverClientManager; +import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl; +import org.apache.hadoop.ozone.common.OzoneChecksumException; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.shell.OzoneAddress; + +/** + * Verifies the checksum of blocks by checking each replica associated + * with a given key. + */ +public class ChecksumVerifier implements ReplicaVerifier { + private final OzoneClient client; + private final OzoneAddress address; + private final OzoneConfiguration conf; + private final ContainerOperationClient containerClient; + private final XceiverClientManager xceiverClientManager; + private static final String CHECK_TYPE = "checksum"; + + @Override + public String getType() { + return CHECK_TYPE; + } + + public ChecksumVerifier(OzoneClient client, OzoneAddress address, OzoneConfiguration conf) throws IOException { + this.client = client; + this.address = address; + this.conf = conf; + this.containerClient = new ContainerOperationClient(conf); + this.xceiverClientManager = containerClient.getXceiverClientManager(); + } + + @Override + public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocationInfo keyLocation) { + Pipeline pipeline = Pipeline.newBuilder(keyLocation.getPipeline()) + .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)) + .setNodes(Collections.singletonList(datanode)) + .build(); + + try (InputStream is = new BlockInputStreamFactoryImpl().create( + keyLocation.getPipeline().getReplicationConfig(), + keyLocation, + pipeline, + keyLocation.getToken(), + xceiverClientManager, + null, + conf.getObject(OzoneClientConfig.class))) { + IOUtils.copyLarge(is, NullOutputStream.INSTANCE); + return BlockVerificationResult.pass(); + } catch (IOException e) { + Throwable cause = e.getCause() != null ? e.getCause() : e; + if (cause instanceof OzoneChecksumException) { + return BlockVerificationResult.failCheck(cause.getMessage()); + } else { + return BlockVerificationResult.failIncomplete(cause.getMessage()); + } + } + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java deleted file mode 100644 index 1011a9f7e138..000000000000 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.debug.replicas; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Collections; -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.output.NullOutputStream; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; - -/** - * Class that downloads every replica for all the blocks associated with a - * given key. It also generates a manifest file with information about the - * downloaded replicas. - */ -public class Checksums implements ReplicaVerifier { - private static final String CHECKTYPE = "checksum"; - - public Checksums() { - } - - @Override - public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OzoneInputStream stream, - OmKeyLocationInfo keyLocation) { - try (InputStream is = stream) { - IOUtils.copyLarge(is, NullOutputStream.INSTANCE); - - return new BlockVerificationResult(CHECKTYPE, true, Collections.emptyList()); - } catch (IOException e) { - BlockVerificationResult.FailureDetail failure = new BlockVerificationResult - .FailureDetail(true, e.getMessage()); - return new BlockVerificationResult(CHECKTYPE, false, Collections.singletonList(failure)); - } - } - -} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java index 8472c5fb07f0..07dae98de1da 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java @@ -18,13 +18,13 @@ package org.apache.hadoop.ozone.debug.replicas; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; /** * Functional interface for implementing a block verifier. */ -@FunctionalInterface public interface ReplicaVerifier { - BlockVerificationResult verifyBlock(DatanodeDetails datanode, OzoneInputStream stream, OmKeyLocationInfo keyLocation); + BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocationInfo keyLocation); + + String getType(); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java index b991f7d24547..582e4abbd6e1 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java @@ -17,14 +17,13 @@ package org.apache.hadoop.ozone.debug.replicas; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Map; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.cli.ScmOption; import org.apache.hadoop.hdds.server.JsonUtils; @@ -33,11 +32,11 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientException; import org.apache.hadoop.ozone.client.OzoneKey; -import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.io.OzoneInputStream; -import org.apache.hadoop.ozone.client.protocol.ClientProtocol; +import org.apache.hadoop.ozone.client.rpc.RpcClient; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.shell.Handler; import org.apache.hadoop.ozone.shell.OzoneAddress; import org.apache.hadoop.ozone.shell.Shell; @@ -63,9 +62,9 @@ public class ReplicasVerify extends Handler { required = true) private String outputDir; - @CommandLine.Option(names = {"--all"}, + @CommandLine.Option(names = {"--all-results"}, description = "Print results for all passing and failing keys") - private boolean allKeys; + private boolean allResults; @CommandLine.ArgGroup(exclusive = false, multiplicity = "1") private Verification verification; @@ -90,7 +89,7 @@ protected void execute(OzoneClient client, OzoneAddress address) throws IOExcept replicaVerifiers = new ArrayList<>(); if (verification.doExecuteChecksums) { - replicaVerifiers.add(new Checksums()); + replicaVerifiers.add(new ChecksumVerifier(client, address, getConf())); } if (verification.doExecuteBlockExistence) { @@ -111,13 +110,12 @@ void findCandidateKeys(OzoneClient ozoneClient, OzoneAddress address) throws IOE String bucketName = address.getBucketName(); String keyName = address.getKeyName(); - ObjectMapper mapper = new ObjectMapper(); - ObjectNode root = mapper.createObjectNode(); - ArrayNode keysArray = mapper.createArrayNode(); + ObjectNode root = JsonUtils.createObjectNode(null); + ArrayNode keysArray = root.putArray("keys"); if (!keyName.isEmpty()) { - OzoneKeyDetails keyDetails = ozoneClient.getProxy().getKeyDetails(volumeName, bucketName, keyName); - processKey(ozoneClient, keyDetails, keysArray); + OmKeyInfo keyInfo = ((RpcClient) ozoneClient.getProxy()).getKeyInfo(volumeName, bucketName, keyName, false); + processKey(ozoneClient, keyInfo, keysArray); } else if (!bucketName.isEmpty()) { OzoneVolume volume = objectStore.getVolume(volumeName); OzoneBucket bucket = volume.getBucket(bucketName); @@ -131,8 +129,7 @@ void findCandidateKeys(OzoneClient ozoneClient, OzoneAddress address) throws IOE } } - root.set("keys", keysArray); - System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(root)); + System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(root)); } void checkVolume(OzoneClient ozoneClient, OzoneVolume volume, ArrayNode keysArray) throws IOException { @@ -147,33 +144,28 @@ void checkBucket(OzoneClient ozoneClient, OzoneBucket bucket, ArrayNode keysArra OzoneKey key = it.next(); // TODO: Remove this check once HDDS-12094 is fixed if (!key.getName().endsWith("/")) { - processKey(ozoneClient, bucket.getKey(key.getName()), keysArray); + OmKeyInfo keyInfo = ((RpcClient) ozoneClient.getProxy()).getKeyInfo( + bucket.getVolumeName(), bucket.getName(), key.getName(), false); + processKey(ozoneClient, keyInfo, keysArray); } } } - void processKey(OzoneClient ozoneClient, OzoneKeyDetails keyDetails, ArrayNode keysArray) { - ObjectMapper mapper = new ObjectMapper(); - - String volumeName = keyDetails.getVolumeName(); - String bucketName = keyDetails.getBucketName(); - String keyName = keyDetails.getName(); + void processKey(OzoneClient ozoneClient, OmKeyInfo keyInfo, ArrayNode keysArray) { + String volumeName = keyInfo.getVolumeName(); + String bucketName = keyInfo.getBucketName(); + String keyName = keyInfo.getKeyName(); ObjectNode keyNode = JsonUtils.createObjectNode(null); keyNode.put("volumeName", volumeName); keyNode.put("bucketName", bucketName); keyNode.put("name", keyName); - ArrayNode blocksArray = mapper.createArrayNode(); - boolean keyPass = true; //to check if all checks are passed - - try { - ClientProtocol clientProtocol = ozoneClient.getObjectStore().getClientProxy(); - Map> replicas = - clientProtocol.getKeysEveryReplicas(volumeName, bucketName, keyName); + ArrayNode blocksArray = keyNode.putArray("blocks"); + boolean keyPass = true; - for (Map.Entry> block : replicas.entrySet()) { - OmKeyLocationInfo keyLocation = block.getKey(); + for (OmKeyLocationInfoGroup keyLocationInfoGroup : keyInfo.getKeyLocationVersions()) { + for (OmKeyLocationInfo keyLocation : keyLocationInfoGroup.getLocationList()) { long containerID = keyLocation.getContainerID(); long localID = keyLocation.getLocalID(); @@ -181,56 +173,55 @@ void processKey(OzoneClient ozoneClient, OzoneKeyDetails keyDetails, ArrayNode k blockNode.put("containerID", containerID); blockNode.put("localID", localID); - ArrayNode replicasArray = mapper.createArrayNode(); - boolean blockPass = !block.getValue().isEmpty(); // If no replicas, the block automatically fails - - for (Map.Entry replica : block.getValue().entrySet()) { - DatanodeDetails datanode = replica.getKey(); + ArrayNode replicasArray = blockNode.putArray("replicas"); + boolean blockPass = true; + for (DatanodeDetails datanode : keyLocation.getPipeline().getNodes()) { ObjectNode datanodeNode = JsonUtils.createObjectNode(null); datanodeNode.put("uuid", datanode.getUuidString()); datanodeNode.put("hostname", datanode.getHostName()); - - ArrayNode checksArray = mapper.createArrayNode(); + ArrayNode checksArray = JsonUtils.createArrayNode(); boolean replicaPass = true; - ObjectMapper replicaVerifierNode = new ObjectMapper(); for (ReplicaVerifier verifier : replicaVerifiers) { - BlockVerificationResult result = verifier.verifyBlock(replica.getKey(), replica.getValue(), keyLocation); - ObjectNode checksNode = result.toJson(replicaVerifierNode); - checksArray.add(checksNode); + BlockVerificationResult result = verifier.verifyBlock(datanode, keyLocation); + ObjectNode checkNode = JsonUtils.createObjectNode(null); + checkNode.put("type", verifier.getType()); + checkNode.put("pass", result.passed()); + + ArrayNode failuresArray = checkNode.putArray("failures"); + for (BlockVerificationResult.FailureDetail failure : result.getFailures().orElse(Collections.emptyList())) { + ObjectNode failureNode = JsonUtils.createObjectNode(null); + failureNode.put("completed", failure.isCompleted()); + failureNode.put("message", failure.getFailureMessage()); + failuresArray.add(failureNode); + } + checksArray.add(checkNode); - if (!result.isPass()) { + if (!result.passed()) { replicaPass = false; } } - ObjectNode replicaNode = mapper.createObjectNode(); + ObjectNode replicaNode = JsonUtils.createObjectNode(null); replicaNode.set("datanode", datanodeNode); replicaNode.set("checks", checksArray); - //replicaNode.put("pass", replicaPass); // ← include pass flag for replica if needed - replicasArray.add(replicaNode); if (!replicaPass) { blockPass = false; } } - blockNode.set("replicas", replicasArray); - // blockNode.put("pass", blockPass); // ← pass flag per block - blocksArray.add(blockNode); if (!blockPass) { keyPass = false; } } - } catch (IOException e) { - throw new RuntimeException("Error retrieving replicas for key: " + keyName); } - if (!keyPass || allKeys) { + if (!keyPass || allResults) { keyNode.set("blocks", blocksArray); keyNode.put("pass", keyPass); keysArray.add(keyNode); From 3671a2a87180ec840f4d0984ec9e095454bf25d7 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Mon, 21 Apr 2025 22:03:52 +0530 Subject: [PATCH 04/13] Fixed pmd errors --- .../hadoop/ozone/debug/replicas/BlockExistenceVerifier.java | 2 -- .../hadoop/ozone/debug/replicas/ChecksumVerifier.java | 6 +----- .../apache/hadoop/ozone/debug/replicas/ReplicasVerify.java | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java index c34950b62356..66eddbb8af1f 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java @@ -36,7 +36,6 @@ * Verifies block existence by making getBlock calls to the datanode. */ public class BlockExistenceVerifier implements ReplicaVerifier { - private final OzoneConfiguration conf; private final ContainerOperationClient containerClient; private final XceiverClientManager xceiverClientManager; private static final String CHECK_TYPE = "blockExistence"; @@ -47,7 +46,6 @@ public String getType() { } public BlockExistenceVerifier(OzoneConfiguration conf) throws IOException { - this.conf = conf; this.containerClient = new ContainerOperationClient(conf); this.xceiverClientManager = containerClient.getXceiverClientManager(); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java index df25085a8818..7b268a62bd7d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java @@ -42,8 +42,6 @@ * with a given key. */ public class ChecksumVerifier implements ReplicaVerifier { - private final OzoneClient client; - private final OzoneAddress address; private final OzoneConfiguration conf; private final ContainerOperationClient containerClient; private final XceiverClientManager xceiverClientManager; @@ -54,9 +52,7 @@ public String getType() { return CHECK_TYPE; } - public ChecksumVerifier(OzoneClient client, OzoneAddress address, OzoneConfiguration conf) throws IOException { - this.client = client; - this.address = address; + public ChecksumVerifier(OzoneConfiguration conf) throws IOException { this.conf = conf; this.containerClient = new ContainerOperationClient(conf); this.xceiverClientManager = containerClient.getXceiverClientManager(); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java index 582e4abbd6e1..7e0f8a839b8a 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java @@ -89,7 +89,7 @@ protected void execute(OzoneClient client, OzoneAddress address) throws IOExcept replicaVerifiers = new ArrayList<>(); if (verification.doExecuteChecksums) { - replicaVerifiers.add(new ChecksumVerifier(client, address, getConf())); + replicaVerifiers.add(new ChecksumVerifier(getConf())); } if (verification.doExecuteBlockExistence) { From 7606334cee3acc29bd079e81609e604267722ea5 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Mon, 21 Apr 2025 22:28:34 +0530 Subject: [PATCH 05/13] Removed unused imports --- .../apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java index 7b268a62bd7d..b937191d1869 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java @@ -31,11 +31,9 @@ import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; -import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl; import org.apache.hadoop.ozone.common.OzoneChecksumException; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.shell.OzoneAddress; /** * Verifies the checksum of blocks by checking each replica associated From 5175151317102c1a5aa586b5121e323c8d66750e Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 22 Apr 2025 12:18:09 +0530 Subject: [PATCH 06/13] Added a test and restructured some methods --- .../ozone/client/protocol/ClientProtocol.java | 12 ++ .../ozone/shell/TestOzoneDebugShell.java | 22 +++ .../ozone/client/ClientProtocolStub.java | 8 ++ .../replicas/BlockExistenceVerifier.java | 2 +- .../replicas/BlockVerificationResult.java | 47 ++----- .../ozone/debug/replicas/ReplicasVerify.java | 131 +++++++++--------- 6 files changed, 118 insertions(+), 104 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 7ef2c38eb32b..c2ce5be16712 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -51,6 +51,7 @@ import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; @@ -436,6 +437,17 @@ OzoneDataStreamOutput createStreamKey(String volumeName, String bucketName, OzoneInputStream getKey(String volumeName, String bucketName, String keyName) throws IOException; + /** + * Reads key info from an existing bucket. + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param forceUpdateContainerCache if true force OM to update container cache location from SCM + * @return {@link OmKeyInfo} + * @throws IOException + */ + OmKeyInfo getKeyInfo(String volumeName, String bucketName, String keyName, + boolean forceUpdateContainerCache) throws IOException; /** * Deletes an existing key. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index a1ace9f97f38..187f25690de8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java @@ -113,6 +113,28 @@ public static void init() throws Exception { startCluster(); } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testReplicasVerifyCmd(boolean isEcKey) throws Exception { + final String volumeName = UUID.randomUUID().toString(); + final String bucketName = UUID.randomUUID().toString(); + final String keyName = UUID.randomUUID().toString(); + + writeKey(volumeName, bucketName, keyName, isEcKey); + + String bucketPath = Path.SEPARATOR + volumeName + Path.SEPARATOR + bucketName; + String fullKeyPath = bucketPath + Path.SEPARATOR + keyName; + + String[] args = new String[] { + getSetConfStringFromConf(OMConfigKeys.OZONE_OM_ADDRESS_KEY), + "replicas", "verify", "--checksums", "--block-existence", fullKeyPath, "--output-dir", "/"//, "--all-results" + }; + + int exitCode = ozoneDebugShell.execute(args); + assertEquals(0, exitCode); + } + + @ParameterizedTest @ValueSource(booleans = {true, false}) public void testChunkInfoCmdBeforeAfterCloseContainer(boolean isEcKey) throws Exception { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index c15a85dc2e36..af1152cb0fc9 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; @@ -253,6 +254,13 @@ public OzoneInputStream getKey(String volumeName, String bucketName, return getBucket(volumeName, bucketName).readKey(keyName); } + @Override + public OmKeyInfo getKeyInfo(String volumeName, String bucketName, String keyName, + boolean forceUpdateContainerCache) throws IOException { + return objectStoreStub.getClientProxy().getKeyInfo( + volumeName, bucketName, keyName, forceUpdateContainerCache); + } + private OzoneBucket getBucket(String volumeName, String bucketName) throws IOException { return objectStoreStub.getVolume(volumeName).getBucket(bucketName); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java index 66eddbb8af1f..dfb049495ec5 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java @@ -63,7 +63,7 @@ public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocati client, keyLocation.getBlockID(), keyLocation.getToken(), - Collections.singletonMap(datanode, 1) + pipeline.getReplicaIndexes() ); boolean hasBlock = response != null && response.hasBlockData(); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java index 50702d84fadf..c73635ba4755 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java @@ -19,65 +19,44 @@ import java.util.Collections; import java.util.List; -import java.util.Optional; /** * Json structure for replicas to pass through each check and give output. */ public class BlockVerificationResult { + private final boolean completed; private final boolean pass; - private final List failures; + private final List failures; - public BlockVerificationResult(boolean pass, List failures) { + public BlockVerificationResult(boolean completed, boolean pass, List failures) { + this.completed = completed; this.pass = pass; this.failures = failures; } public static BlockVerificationResult pass() { - return new BlockVerificationResult(true, null); + return new BlockVerificationResult(true, true, Collections.emptyList()); } public static BlockVerificationResult failCheck(String message) { - return new BlockVerificationResult(false, - Collections.singletonList(new FailureDetail(true, message))); + return new BlockVerificationResult(true, false, Collections.singletonList(message)); } public static BlockVerificationResult failIncomplete(String message) { - return new BlockVerificationResult(false, - Collections.singletonList(new FailureDetail(false, message))); + return new BlockVerificationResult(false, false, Collections.singletonList(message)); } - public boolean passed() { - return pass; + public boolean isCompleted() { + return completed; } - public Optional> getFailures() { - return Optional.ofNullable(failures); + public boolean passed() { + return pass; } - /** - * Details about the check failure. - */ - public static class FailureDetail { - // indicates whether the check finished and failed, - // or it was unable to finish due to connection or other issues - private final boolean completed; - private final String message; - - public FailureDetail(boolean completed, String message) { - this.completed = completed; - this.message = message; - } - - public boolean isCompleted() { - return completed; - } - - public String getFailureMessage() { - return message; - } - + public List getFailures() { + return failures; } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java index 7e0f8a839b8a..ab7d25682d8c 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.Iterator; import java.util.List; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -33,10 +32,8 @@ import org.apache.hadoop.ozone.client.OzoneClientException; import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; -import org.apache.hadoop.ozone.client.rpc.RpcClient; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.shell.Handler; import org.apache.hadoop.ozone.shell.OzoneAddress; import org.apache.hadoop.ozone.shell.Shell; @@ -113,48 +110,49 @@ void findCandidateKeys(OzoneClient ozoneClient, OzoneAddress address) throws IOE ObjectNode root = JsonUtils.createObjectNode(null); ArrayNode keysArray = root.putArray("keys"); + boolean[] allKeysPassed = {true}; + if (!keyName.isEmpty()) { - OmKeyInfo keyInfo = ((RpcClient) ozoneClient.getProxy()).getKeyInfo(volumeName, bucketName, keyName, false); - processKey(ozoneClient, keyInfo, keysArray); + processKey(ozoneClient, volumeName, bucketName, keyName, keysArray, allKeysPassed); } else if (!bucketName.isEmpty()) { OzoneVolume volume = objectStore.getVolume(volumeName); OzoneBucket bucket = volume.getBucket(bucketName); - checkBucket(ozoneClient, bucket, keysArray); + checkBucket(ozoneClient, bucket, keysArray, allKeysPassed); } else if (!volumeName.isEmpty()) { OzoneVolume volume = objectStore.getVolume(volumeName); - checkVolume(ozoneClient, volume, keysArray); + checkVolume(ozoneClient, volume, keysArray, allKeysPassed); } else { for (Iterator it = objectStore.listVolumes(null); it.hasNext();) { - checkVolume(ozoneClient, it.next(), keysArray); + checkVolume(ozoneClient, it.next(), keysArray, allKeysPassed); } } - + root.put("pass", allKeysPassed[0]); System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(root)); } - void checkVolume(OzoneClient ozoneClient, OzoneVolume volume, ArrayNode keysArray) throws IOException { + void checkVolume(OzoneClient ozoneClient, OzoneVolume volume, ArrayNode keysArray, boolean[] allKeysPassed) + throws IOException { for (Iterator it = volume.listBuckets(null); it.hasNext();) { OzoneBucket bucket = it.next(); - checkBucket(ozoneClient, bucket, keysArray); + checkBucket(ozoneClient, bucket, keysArray, allKeysPassed); } } - void checkBucket(OzoneClient ozoneClient, OzoneBucket bucket, ArrayNode keysArray) throws IOException { + void checkBucket(OzoneClient ozoneClient, OzoneBucket bucket, ArrayNode keysArray, boolean[] allKeysPassed) + throws IOException { for (Iterator it = bucket.listKeys(null); it.hasNext();) { OzoneKey key = it.next(); // TODO: Remove this check once HDDS-12094 is fixed if (!key.getName().endsWith("/")) { - OmKeyInfo keyInfo = ((RpcClient) ozoneClient.getProxy()).getKeyInfo( - bucket.getVolumeName(), bucket.getName(), key.getName(), false); - processKey(ozoneClient, keyInfo, keysArray); + processKey(ozoneClient, key.getVolumeName(), key.getBucketName(), key.getName(), keysArray, allKeysPassed); } } } - void processKey(OzoneClient ozoneClient, OmKeyInfo keyInfo, ArrayNode keysArray) { - String volumeName = keyInfo.getVolumeName(); - String bucketName = keyInfo.getBucketName(); - String keyName = keyInfo.getKeyName(); + void processKey(OzoneClient ozoneClient, String volumeName, String bucketName, String keyName, + ArrayNode keysArray, boolean[] allKeysPassed) throws IOException { + OmKeyInfo keyInfo = ozoneClient.getProxy().getKeyInfo( + volumeName, bucketName, keyName, false); ObjectNode keyNode = JsonUtils.createObjectNode(null); keyNode.put("volumeName", volumeName); @@ -164,66 +162,61 @@ void processKey(OzoneClient ozoneClient, OmKeyInfo keyInfo, ArrayNode keysArray) ArrayNode blocksArray = keyNode.putArray("blocks"); boolean keyPass = true; - for (OmKeyLocationInfoGroup keyLocationInfoGroup : keyInfo.getKeyLocationVersions()) { - for (OmKeyLocationInfo keyLocation : keyLocationInfoGroup.getLocationList()) { - long containerID = keyLocation.getContainerID(); - long localID = keyLocation.getLocalID(); - - ObjectNode blockNode = JsonUtils.createObjectNode(null); - blockNode.put("containerID", containerID); - blockNode.put("localID", localID); - - ArrayNode replicasArray = blockNode.putArray("replicas"); - boolean blockPass = true; - - for (DatanodeDetails datanode : keyLocation.getPipeline().getNodes()) { - ObjectNode datanodeNode = JsonUtils.createObjectNode(null); - datanodeNode.put("uuid", datanode.getUuidString()); - datanodeNode.put("hostname", datanode.getHostName()); - - ArrayNode checksArray = JsonUtils.createArrayNode(); - boolean replicaPass = true; - - for (ReplicaVerifier verifier : replicaVerifiers) { - BlockVerificationResult result = verifier.verifyBlock(datanode, keyLocation); - ObjectNode checkNode = JsonUtils.createObjectNode(null); - checkNode.put("type", verifier.getType()); - checkNode.put("pass", result.passed()); - - ArrayNode failuresArray = checkNode.putArray("failures"); - for (BlockVerificationResult.FailureDetail failure : result.getFailures().orElse(Collections.emptyList())) { - ObjectNode failureNode = JsonUtils.createObjectNode(null); - failureNode.put("completed", failure.isCompleted()); - failureNode.put("message", failure.getFailureMessage()); - failuresArray.add(failureNode); - } - checksArray.add(checkNode); - - if (!result.passed()) { - replicaPass = false; - } - } + for (OmKeyLocationInfo keyLocation : keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly()) { + long containerID = keyLocation.getContainerID(); + long localID = keyLocation.getLocalID(); + + ObjectNode blockNode = blocksArray.addObject(); + blockNode.put("containerID", containerID); + blockNode.put("blockID", localID); + + ArrayNode replicasArray = blockNode.putArray("replicas"); + boolean blockPass = true; + + for (DatanodeDetails datanode : keyLocation.getPipeline().getNodes()) { + ObjectNode replicaNode = replicasArray.addObject(); + + ObjectNode datanodeNode = replicaNode.putObject("datanode"); + datanodeNode.put("uuid", datanode.getUuidString()); + datanodeNode.put("hostname", datanode.getHostName()); - ObjectNode replicaNode = JsonUtils.createObjectNode(null); - replicaNode.set("datanode", datanodeNode); - replicaNode.set("checks", checksArray); - replicasArray.add(replicaNode); + ArrayNode checksArray = replicaNode.putArray("checks"); + boolean replicaPass = true; - if (!replicaPass) { - blockPass = false; + for (ReplicaVerifier verifier : replicaVerifiers) { + BlockVerificationResult result = verifier.verifyBlock(datanode, keyLocation); + ObjectNode checkNode = checksArray.addObject(); + checkNode.put("type", verifier.getType()); + checkNode.put("completed", result.isCompleted()); + checkNode.put("pass", result.passed()); + + ArrayNode failuresArray = checkNode.putArray("failures"); + for (String failure : result.getFailures()) { + failuresArray.addObject().put("message", failure); + } + replicaNode.put("replicaIndex", keyLocation.getPipeline().getReplicaIndex(datanode)); + + if (!result.passed()) { + replicaPass = false; } } - blocksArray.add(blockNode); - if (!blockPass) { - keyPass = false; + if (!replicaPass) { + blockPass = false; } } + + if (!blockPass) { + keyPass = false; + } + } + + keyNode.put("pass", keyPass); + if (!keyPass) { + allKeysPassed[0] = false; } if (!keyPass || allResults) { - keyNode.set("blocks", blocksArray); - keyNode.put("pass", keyPass); keysArray.add(keyNode); } } From aecdc777897aa298bf8b26ce17473b3d593b8890 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 22 Apr 2025 12:32:56 +0530 Subject: [PATCH 07/13] PMD failure added Override annotation --- .../main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 11d96393c04c..4f9d486b4bab 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1850,6 +1850,7 @@ private OmKeyInfo getS3PartKeyInfo( return keyInfoWithS3Context.getKeyInfo(); } + @Override public OmKeyInfo getKeyInfo( String volumeName, String bucketName, String keyName, boolean forceUpdateContainerCache) throws IOException { From 7b4f472a0df3d54bc5831cf5b3c7fdfea4ea0660 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Wed, 23 Apr 2025 15:52:21 +0530 Subject: [PATCH 08/13] Added a robot test and inclued replicaIndex in verifyBlock() --- .../dist/src/main/compose/common/replicas-test.sh | 1 + .../src/main/smoketest/debug/ozone-debug-tests.robot | 12 ++++++------ .../ozone/debug/replicas/BlockExistenceVerifier.java | 4 +++- .../ozone/debug/replicas/ChecksumVerifier.java | 4 +++- .../hadoop/ozone/debug/replicas/ReplicaVerifier.java | 2 +- .../hadoop/ozone/debug/replicas/ReplicasVerify.java | 5 +++-- 6 files changed, 17 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh index 485e189eed5b..b875bfcafc8e 100755 --- a/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/replicas-test.sh @@ -23,6 +23,7 @@ key="testfile" execute_robot_test ${SCM} -v "PREFIX:${prefix}" debug/ozone-debug-tests.robot +#TODO HDDS-12890: Add acceptance robot tests for ozone debug replicas verify # get block locations for key #chunkinfo="${key}-blocks-${prefix}" #docker-compose exec -T ${SCM} bash -c "ozone debug replicas chunk-info ${volume}/${bucket}/${key}" > "$chunkinfo" diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot index d64d0c0c9744..a0149cde6f2e 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot @@ -36,13 +36,13 @@ Write keys Execute ozone sh key put o3://om/${VOLUME}/${BUCKET}/${TESTFILE} ${TEMP_DIR}/${TESTFILE} *** Test Cases *** -#Test ozone debug read-replicas -# ${directory} = Execute replicas verify checksums CLI tool -# Set Test Variable ${DIR} ${directory} - -# ${count_files} = Count Files In Directory ${directory} -# Should Be Equal As Integers ${count_files} 1 +Test ozone debug replicas verify checksums + ${output} = Execute ozone debug replicas verify --checksums / --output-dir ${TEMP_DIR} + ${json} = Evaluate json.loads('''${output}''') json + # 'keys' array should be empty if all keys and their replicas passed checksum verification + Should Be Empty ${json}[keys] + Should Be True ${json}[pass] ${True} Test ozone debug version ${output} = Execute ozone debug version diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java index dfb049495ec5..f1d77b8544e6 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java @@ -51,11 +51,13 @@ public BlockExistenceVerifier(OzoneConfiguration conf) throws IOException { } @Override - public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocationInfo keyLocation) { + public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocationInfo keyLocation, + int replicaIndex) { try { Pipeline pipeline = Pipeline.newBuilder(keyLocation.getPipeline()) .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)) .setNodes(Collections.singletonList(datanode)) + .setReplicaIndexes(Collections.singletonMap(datanode, replicaIndex)) .build(); XceiverClientSpi client = xceiverClientManager.acquireClientForReadData(pipeline); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java index b937191d1869..0f4729fbae27 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java @@ -57,10 +57,12 @@ public ChecksumVerifier(OzoneConfiguration conf) throws IOException { } @Override - public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocationInfo keyLocation) { + public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocationInfo keyLocation, + int replicaIndex) { Pipeline pipeline = Pipeline.newBuilder(keyLocation.getPipeline()) .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)) .setNodes(Collections.singletonList(datanode)) + .setReplicaIndexes(Collections.singletonMap(datanode, replicaIndex)) .build(); try (InputStream is = new BlockInputStreamFactoryImpl().create( diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java index 07dae98de1da..af9bcdac4b83 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicaVerifier.java @@ -24,7 +24,7 @@ * Functional interface for implementing a block verifier. */ public interface ReplicaVerifier { - BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocationInfo keyLocation); + BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocationInfo keyLocation, int replicaIndex); String getType(); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java index ab7d25682d8c..02c30de33a6b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java @@ -182,9 +182,10 @@ void processKey(OzoneClient ozoneClient, String volumeName, String bucketName, S ArrayNode checksArray = replicaNode.putArray("checks"); boolean replicaPass = true; + int replicaIndex = keyLocation.getPipeline().getReplicaIndex(datanode); for (ReplicaVerifier verifier : replicaVerifiers) { - BlockVerificationResult result = verifier.verifyBlock(datanode, keyLocation); + BlockVerificationResult result = verifier.verifyBlock(datanode, keyLocation, replicaIndex); ObjectNode checkNode = checksArray.addObject(); checkNode.put("type", verifier.getType()); checkNode.put("completed", result.isCompleted()); @@ -194,7 +195,7 @@ void processKey(OzoneClient ozoneClient, String volumeName, String bucketName, S for (String failure : result.getFailures()) { failuresArray.addObject().put("message", failure); } - replicaNode.put("replicaIndex", keyLocation.getPipeline().getReplicaIndex(datanode)); + replicaNode.put("replicaIndex", replicaIndex); if (!result.passed()) { replicaPass = false; From 8e5da41d2c9c2f1383e7a1ec78757823ecc9127a Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Wed, 23 Apr 2025 17:43:47 +0530 Subject: [PATCH 09/13] Specified the key path to run the command in robot test --- .../dist/src/main/smoketest/debug/ozone-debug-tests.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot index a0149cde6f2e..97fe36d2c50e 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests.robot @@ -37,7 +37,7 @@ Write keys *** Test Cases *** Test ozone debug replicas verify checksums - ${output} = Execute ozone debug replicas verify --checksums / --output-dir ${TEMP_DIR} + ${output} = Execute ozone debug replicas verify --checksums o3://om/${VOLUME}/${BUCKET}/${TESTFILE} --output-dir ${TEMP_DIR} ${json} = Evaluate json.loads('''${output}''') json # 'keys' array should be empty if all keys and their replicas passed checksum verification From 33fc833f2d56b42b199315d82aa1448957ce5b98 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 May 2025 09:29:19 +0530 Subject: [PATCH 10/13] Delete Checksums and close client in BlockExistenceVerifier --- .../apache/hadoop/ozone/shell/TestOzoneDebugShell.java | 3 ++- .../ozone/debug/replicas/BlockExistenceVerifier.java | 8 ++++++-- .../org/apache/hadoop/ozone/debug/replicas/Checksums.java | 0 3 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index 6d6936e30dbb..203e13c8868d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java @@ -94,11 +94,12 @@ public void testReplicasVerifyCmd(boolean isEcKey) throws Exception { final String bucketName = UUID.randomUUID().toString(); final String keyName = UUID.randomUUID().toString(); - writeKey(volumeName, bucketName, keyName, isEcKey); + writeKey(volumeName, bucketName, keyName, isEcKey, BucketLayout.FILE_SYSTEM_OPTIMIZED); String bucketPath = Path.SEPARATOR + volumeName + Path.SEPARATOR + bucketName; String fullKeyPath = bucketPath + Path.SEPARATOR + keyName; + //TODO HDDS-12715: Create common integration test cluster for debug and repair tools String[] args = new String[] { getSetConfStringFromConf(OMConfigKeys.OZONE_OM_ADDRESS_KEY), "replicas", "verify", "--checksums", "--block-existence", fullKeyPath, "--output-dir", "/"//, "--all-results" diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java index f1d77b8544e6..01a344c6034b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java @@ -52,7 +52,8 @@ public BlockExistenceVerifier(OzoneConfiguration conf) throws IOException { @Override public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocationInfo keyLocation, - int replicaIndex) { + int replicaIndex) { + XceiverClientSpi client = null; try { Pipeline pipeline = Pipeline.newBuilder(keyLocation.getPipeline()) .setReplicationConfig(StandaloneReplicationConfig.getInstance(ONE)) @@ -60,7 +61,7 @@ public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocati .setReplicaIndexes(Collections.singletonMap(datanode, replicaIndex)) .build(); - XceiverClientSpi client = xceiverClientManager.acquireClientForReadData(pipeline); + client = xceiverClientManager.acquireClientForReadData(pipeline); ContainerProtos.GetBlockResponseProto response = ContainerProtocolCalls.getBlock( client, keyLocation.getBlockID(), @@ -78,5 +79,8 @@ public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocati } catch (IOException e) { return BlockVerificationResult.failIncomplete(e.getMessage()); } + finally { + xceiverClientManager.releaseClient(client, false); + } } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/Checksums.java deleted file mode 100644 index e69de29bb2d1..000000000000 From 10c7bcb8cc47e7f9a9ade36428d8c9b9ebf78b19 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 May 2025 09:41:30 +0530 Subject: [PATCH 11/13] Fixed checkstyle errors --- .../org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java | 1 - .../hadoop/ozone/debug/replicas/BlockExistenceVerifier.java | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index 203e13c8868d..65b1283015ce 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java @@ -109,7 +109,6 @@ public void testReplicasVerifyCmd(boolean isEcKey) throws Exception { assertEquals(0, exitCode); } - @ParameterizedTest @ValueSource(booleans = {true, false}) public void testChunkInfoCmdBeforeAfterCloseContainer(boolean isEcKey) throws Exception { diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java index 01a344c6034b..9d910700ea40 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java @@ -78,8 +78,7 @@ public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocati } } catch (IOException e) { return BlockVerificationResult.failIncomplete(e.getMessage()); - } - finally { + } finally { xceiverClientManager.releaseClient(client, false); } } From 474d3970705abbc416802144f1c34f6878d3631e Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 May 2025 10:32:33 +0530 Subject: [PATCH 12/13] Used AtomicBoolean and fixed the test --- .../hadoop/ozone/shell/TestOzoneDebugShell.java | 1 + .../hadoop/ozone/debug/replicas/ReplicasVerify.java | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java index 65b1283015ce..eb5c87fabb04 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java @@ -102,6 +102,7 @@ public void testReplicasVerifyCmd(boolean isEcKey) throws Exception { //TODO HDDS-12715: Create common integration test cluster for debug and repair tools String[] args = new String[] { getSetConfStringFromConf(OMConfigKeys.OZONE_OM_ADDRESS_KEY), + getSetConfStringFromConf(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY), "replicas", "verify", "--checksums", "--block-existence", fullKeyPath, "--output-dir", "/"//, "--all-results" }; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java index c5f12dcd8bdc..38ebee0e8ac0 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.cli.ScmOption; import org.apache.hadoop.hdds.server.JsonUtils; @@ -97,7 +98,7 @@ void findCandidateKeys(OzoneClient ozoneClient, OzoneAddress address) throws IOE ObjectNode root = JsonUtils.createObjectNode(null); ArrayNode keysArray = root.putArray("keys"); - boolean[] allKeysPassed = {true}; + AtomicBoolean allKeysPassed = new AtomicBoolean(true); if (!keyName.isEmpty()) { processKey(ozoneClient, volumeName, bucketName, keyName, keysArray, allKeysPassed); @@ -113,11 +114,11 @@ void findCandidateKeys(OzoneClient ozoneClient, OzoneAddress address) throws IOE checkVolume(ozoneClient, it.next(), keysArray, allKeysPassed); } } - root.put("pass", allKeysPassed[0]); + root.put("pass", allKeysPassed.get()); System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(root)); } - void checkVolume(OzoneClient ozoneClient, OzoneVolume volume, ArrayNode keysArray, boolean[] allKeysPassed) + void checkVolume(OzoneClient ozoneClient, OzoneVolume volume, ArrayNode keysArray, AtomicBoolean allKeysPassed) throws IOException { for (Iterator it = volume.listBuckets(null); it.hasNext();) { OzoneBucket bucket = it.next(); @@ -125,7 +126,7 @@ void checkVolume(OzoneClient ozoneClient, OzoneVolume volume, ArrayNode keysArra } } - void checkBucket(OzoneClient ozoneClient, OzoneBucket bucket, ArrayNode keysArray, boolean[] allKeysPassed) + void checkBucket(OzoneClient ozoneClient, OzoneBucket bucket, ArrayNode keysArray, AtomicBoolean allKeysPassed) throws IOException { for (Iterator it = bucket.listKeys(null); it.hasNext();) { OzoneKey key = it.next(); @@ -137,7 +138,7 @@ void checkBucket(OzoneClient ozoneClient, OzoneBucket bucket, ArrayNode keysArra } void processKey(OzoneClient ozoneClient, String volumeName, String bucketName, String keyName, - ArrayNode keysArray, boolean[] allKeysPassed) throws IOException { + ArrayNode keysArray, AtomicBoolean allKeysPassed) throws IOException { OmKeyInfo keyInfo = ozoneClient.getProxy().getKeyInfo( volumeName, bucketName, keyName, false); @@ -201,7 +202,7 @@ void processKey(OzoneClient ozoneClient, String volumeName, String bucketName, S keyNode.put("pass", keyPass); if (!keyPass) { - allKeysPassed[0] = false; + allKeysPassed.set(false); } if (!keyPass || allResults) { From 1bbf68c3ed5b123c589339df8842e0b0f51fae8b Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 6 May 2025 14:49:07 +0530 Subject: [PATCH 13/13] Changed ozonefs-obs.robot volume name --- hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot index c221c59f99ae..f6491ceebf67 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs-obs.robot @@ -23,7 +23,7 @@ Test Timeout 5 minutes *** Variables *** ${SCHEME} ofs -${volume} volume1 +${volume} obs-volume1 ${bucket} obs-bucket1 ${PREFIX} ozone