From c1d94aad78dee2bd0bd7ec18414ae91d1dd622ba Mon Sep 17 00:00:00 2001 From: sumitagrawl Date: Tue, 20 May 2025 11:00:33 +0530 Subject: [PATCH 1/3] HDDS-12926. remove *.tmp.* exclusion in DU --- .../src/main/java/org/apache/hadoop/hdds/fs/DUFactory.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DUFactory.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DUFactory.java index a89e914398c9..266244763cea 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DUFactory.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/fs/DUFactory.java @@ -31,7 +31,6 @@ public class DUFactory implements SpaceUsageCheckFactory { private static final String DU_CACHE_FILE = "scmUsed"; - private static final String EXCLUDE_PATTERN = "*.tmp.*"; private Conf conf; @@ -46,7 +45,7 @@ public SpaceUsageCheckFactory setConfiguration( public SpaceUsageCheckParams paramsFor(File dir) { Duration refreshPeriod = conf.getRefreshPeriod(); - SpaceUsageSource source = new DU(dir, EXCLUDE_PATTERN); + SpaceUsageSource source = new DU(dir, null); SpaceUsagePersistence persistence = new SaveSpaceUsageToFile( new File(dir, DU_CACHE_FILE), refreshPeriod); From ed3fc9168838c73126bce963a087497617c306f4 Mon Sep 17 00:00:00 2001 From: Sumit Agrawal Date: Mon, 27 Oct 2025 14:54:17 +0530 Subject: [PATCH 2/3] HDDS-13842. revert disable ratis metadata write for SCM --- .../main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 5 ----- .../java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java index a921ffe71677..5860d2523d49 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/RatisUtil.java @@ -197,11 +197,6 @@ private static int setRaftLogProperties(final RaftProperties properties, ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_PURGE_GAP_DEFAULT)); Log.setSegmentCacheNumMax(properties, 2); - // This avoids writing commit metadata to Raft Log, which can be used to recover the - // commit index even if a majority of servers are dead. We don't need this for StorageContainerManager, - // disabling this will avoid the additional disk IO. - Log.setLogMetadataEnabled(properties, false); - return logAppenderQueueByteLimit; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java index ef9ffc03f261..a948b9a7a87a 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java @@ -167,9 +167,7 @@ public CompletableFuture applyTransaction( if (scm.isInSafeMode() && refreshedAfterLeaderReady.get()) { scm.getScmSafeModeManager().refreshAndValidate(); } - final TermIndex appliedTermIndex = TermIndex.valueOf(trx.getLogEntry()); - transactionBuffer.updateLatestTrxInfo(TransactionInfo.valueOf(appliedTermIndex)); - updateLastAppliedTermIndex(appliedTermIndex); + transactionBuffer.updateLatestTrxInfo(TransactionInfo.valueOf(TermIndex.valueOf(trx.getLogEntry()))); } catch (Exception ex) { applyTransactionFuture.completeExceptionally(ex); ExitUtils.terminate(1, ex.getMessage(), ex, StateMachine.LOG); From a171ab6ebfe02260cd63ed53693b81e2f3c6fd77 Mon Sep 17 00:00:00 2001 From: Sumit Agrawal Date: Tue, 28 Oct 2025 11:16:09 +0530 Subject: [PATCH 3/3] fix review comment --- .../java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java index a948b9a7a87a..ef9ffc03f261 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java @@ -167,7 +167,9 @@ public CompletableFuture applyTransaction( if (scm.isInSafeMode() && refreshedAfterLeaderReady.get()) { scm.getScmSafeModeManager().refreshAndValidate(); } - transactionBuffer.updateLatestTrxInfo(TransactionInfo.valueOf(TermIndex.valueOf(trx.getLogEntry()))); + final TermIndex appliedTermIndex = TermIndex.valueOf(trx.getLogEntry()); + transactionBuffer.updateLatestTrxInfo(TransactionInfo.valueOf(appliedTermIndex)); + updateLastAppliedTermIndex(appliedTermIndex); } catch (Exception ex) { applyTransactionFuture.completeExceptionally(ex); ExitUtils.terminate(1, ex.getMessage(), ex, StateMachine.LOG);