From bc2e9acda7b07df053fae9490fc9fb6024aba3e2 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 22 Mar 2024 12:00:20 -0700 Subject: [PATCH 1/5] HDDS-10591. [hsync] improve block token refresh message. Change-Id: If0e713678ee34ccbd74da5f20975bf7bb62fee29 --- .../java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java | 2 +- .../apache/hadoop/hdds/scm/storage/BlockInputStream.java | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java index 52f435dc826d..80a6c5a86b5f 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java @@ -440,7 +440,7 @@ private XceiverClientReply sendCommandWithRetry( LOG.debug(message + " on the pipeline {}.", processForDebug(request), pipeline); } else { - LOG.error(message + " on the pipeline {}.", + LOG.warn(message + " on the pipeline {}.", request.getCmdType(), pipeline); } throw ioException; diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index 2c30abd3d539..5a68fc00ec18 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -217,7 +217,7 @@ private boolean isConnectivityIssue(IOException ex) { } private void refreshBlockInfo(IOException cause) throws IOException { - LOG.info("Unable to read information for block {} from pipeline {}: {}", + LOG.info("Attempting to update pipeline and block token for block {} from pipeline {}: {}", blockID, pipelineRef.get().getId(), cause.getMessage()); if (refreshFunction != null) { LOG.debug("Re-fetching pipeline and block token for block {}", blockID); @@ -225,10 +225,11 @@ private void refreshBlockInfo(IOException cause) throws IOException { if (blockLocationInfo == null) { LOG.debug("No new block location info for block {}", blockID); } else { - LOG.debug("New pipeline for block {}: {}", blockID, - blockLocationInfo.getPipeline()); setPipeline(blockLocationInfo.getPipeline()); tokenRef.set(blockLocationInfo.getToken()); + LOG.info("New pipeline for block {}: {}", blockID, + blockLocationInfo.getPipeline()); + LOG.info("A new token is added {}", blockLocationInfo.getToken()); } } else { throw cause; From d9aab21e9ccf521ef6523b5333b137bd5e64837a Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 1 Apr 2024 11:53:09 -0700 Subject: [PATCH 2/5] Log the expiry of token, not the token itself. Change-Id: I328b7e363b6c8ca2c0b2dc0453a08f34bf139420 --- .../apache/hadoop/hdds/scm/storage/BlockInputStream.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index 5a68fc00ec18..5131d38f6479 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -21,6 +21,7 @@ import java.io.EOFException; import java.io.IOException; import java.io.InputStream; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -223,13 +224,15 @@ private void refreshBlockInfo(IOException cause) throws IOException { LOG.debug("Re-fetching pipeline and block token for block {}", blockID); BlockLocationInfo blockLocationInfo = refreshFunction.apply(blockID); if (blockLocationInfo == null) { - LOG.debug("No new block location info for block {}", blockID); + LOG.warn("No new block location info for block {}", blockID); } else { + OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(); + tokenId.readFromByteArray(blockLocationInfo.getToken().getIdentifier()); setPipeline(blockLocationInfo.getPipeline()); tokenRef.set(blockLocationInfo.getToken()); LOG.info("New pipeline for block {}: {}", blockID, blockLocationInfo.getPipeline()); - LOG.info("A new token is added {}", blockLocationInfo.getToken()); + LOG.info("A new token is added. Expiry: {}", Instant.ofEpochMilli(tokenId.getExpiryDate())); } } else { throw cause; From 7de11e3174f23967271fa753964d7924abd102eb Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 1 Apr 2024 16:19:13 -0700 Subject: [PATCH 3/5] Fix test failure Change-Id: I689c514c0634ef9328f461958bdd4ad24a5d91a6 --- .../hadoop/hdds/scm/storage/BlockInputStream.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index 5131d38f6479..28e30b7252bf 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -226,13 +226,19 @@ private void refreshBlockInfo(IOException cause) throws IOException { if (blockLocationInfo == null) { LOG.warn("No new block location info for block {}", blockID); } else { - OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(); - tokenId.readFromByteArray(blockLocationInfo.getToken().getIdentifier()); setPipeline(blockLocationInfo.getPipeline()); - tokenRef.set(blockLocationInfo.getToken()); LOG.info("New pipeline for block {}: {}", blockID, blockLocationInfo.getPipeline()); - LOG.info("A new token is added. Expiry: {}", Instant.ofEpochMilli(tokenId.getExpiryDate())); + + if (blockLocationInfo.getToken() != null) { + tokenRef.set(blockLocationInfo.getToken()); + OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(); + tokenId.readFromByteArray(tokenRef.get().getIdentifier()); + LOG.info("A new token is added for block {}. Expiry: {}", + blockID, Instant.ofEpochMilli(tokenId.getExpiryDate())); + } else { + LOG.warn("Unable to add a new block token for block {}", blockID); + } } } else { throw cause; From 60cd00825ab3dd2882ea46f6c44ba254be5bd032 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 2 Apr 2024 10:06:34 -0700 Subject: [PATCH 4/5] Remove warning log if token is not returned from OM. This is normal and possible in unsecure environment. Change-Id: I4627fc7ac2fe9ac6f77d22273a80cb40177f488f --- .../org/apache/hadoop/hdds/scm/storage/BlockInputStream.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index 28e30b7252bf..e741d09df90c 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -236,8 +236,6 @@ private void refreshBlockInfo(IOException cause) throws IOException { tokenId.readFromByteArray(tokenRef.get().getIdentifier()); LOG.info("A new token is added for block {}. Expiry: {}", blockID, Instant.ofEpochMilli(tokenId.getExpiryDate())); - } else { - LOG.warn("Unable to add a new block token for block {}", blockID); } } } else { From a3494eb2edc13264df14f5c47ddc9e0de0d7475d Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 4 Apr 2024 10:40:31 -0700 Subject: [PATCH 5/5] Update hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java Co-authored-by: Siyao Meng <50227127+smengcl@users.noreply.github.com> --- .../org/apache/hadoop/hdds/scm/storage/BlockInputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index e741d09df90c..363a26965f68 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -230,8 +230,8 @@ private void refreshBlockInfo(IOException cause) throws IOException { LOG.info("New pipeline for block {}: {}", blockID, blockLocationInfo.getPipeline()); + tokenRef.set(blockLocationInfo.getToken()); if (blockLocationInfo.getToken() != null) { - tokenRef.set(blockLocationInfo.getToken()); OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(); tokenId.readFromByteArray(tokenRef.get().getIdentifier()); LOG.info("A new token is added for block {}. Expiry: {}",