From fdd32b49cee9ff116e643173e19a33aadaf63a20 Mon Sep 17 00:00:00 2001 From: sumitagrawl Date: Thu, 30 Mar 2023 12:23:11 +0530 Subject: [PATCH 1/3] HDDS-8324. DN data cache gets removed randomly asking for data from disk --- .../transport/server/ratis/ContainerStateMachine.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 4883ab9dd205..0d0621a4aec2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -851,7 +851,7 @@ private void removeStateMachineDataIfNeeded(long index) { .getFollowerNextIndices()).min().getAsLong(); LOG.debug("Removing data corresponding to log index {} min index {} " + "from cache", index, minIndex); - stateMachineDataCache.removeIf(k -> k >= (Math.min(minIndex, index))); + stateMachineDataCache.removeIf(k -> k <= (Math.min(minIndex, index))); } } catch (Exception e) { throw new RuntimeException(e); @@ -874,7 +874,7 @@ public CompletableFuture applyTransaction(TransactionContext trx) { // if waitOnBothFollower is false, remove the entry from the cache // as soon as its applied and such entry exists in the cache. if (!waitOnBothFollowers) { - stateMachineDataCache.removeIf(k -> k >= index); + stateMachineDataCache.removeIf(k -> k <= index); } DispatcherContext.Builder builder = new DispatcherContext.Builder().setTerm(trx.getLogEntry().getTerm()) @@ -996,7 +996,7 @@ public void notifyNotLeader(Collection pendingEntries) { @Override public CompletableFuture truncate(long index) { - stateMachineDataCache.removeIf(k -> k >= index); + stateMachineDataCache.removeIf(k -> k <= index); return CompletableFuture.completedFuture(null); } From a9e428ff9bf1e22cbff72b22f7ae55cc65f96b42 Mon Sep 17 00:00:00 2001 From: sumitagrawl Date: Fri, 31 Mar 2023 15:35:57 +0530 Subject: [PATCH 2/3] HDDS-8324. DN data cache gets removed randomly asking for data from disk --- .../server/ratis/ContainerStateMachine.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 0d0621a4aec2..d0271a9b5c7c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -873,9 +873,7 @@ public CompletableFuture applyTransaction(TransactionContext trx) { removeStateMachineDataIfNeeded(index); // if waitOnBothFollower is false, remove the entry from the cache // as soon as its applied and such entry exists in the cache. - if (!waitOnBothFollowers) { - stateMachineDataCache.removeIf(k -> k <= index); - } + removeStateMachineDataIfMajorityFollowSync(index); DispatcherContext.Builder builder = new DispatcherContext.Builder().setTerm(trx.getLogEntry().getTerm()) .setLogIndex(index); @@ -982,6 +980,14 @@ public CompletableFuture applyTransaction(TransactionContext trx) { } } + private void removeStateMachineDataIfMajorityFollowSync(long index) { + if (!waitOnBothFollowers) { + // if majority follow in sync, remove all cache previous to current index + // including current index + stateMachineDataCache.removeIf(k -> k <= index); + } + } + private static CompletableFuture completeExceptionally(Exception e) { final CompletableFuture future = new CompletableFuture<>(); future.completeExceptionally(e); @@ -996,7 +1002,7 @@ public void notifyNotLeader(Collection pendingEntries) { @Override public CompletableFuture truncate(long index) { - stateMachineDataCache.removeIf(k -> k <= index); + stateMachineDataCache.removeIf(k -> k > index); return CompletableFuture.completedFuture(null); } From 6728d364d4667441d8615960423d10fc34260b16 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sat, 1 Apr 2023 21:56:11 +0200 Subject: [PATCH 3/3] Extract method removeCacheDataUpTo --- .../transport/server/ratis/ContainerStateMachine.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index d0271a9b5c7c..7d7b28999fb3 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -851,7 +851,7 @@ private void removeStateMachineDataIfNeeded(long index) { .getFollowerNextIndices()).min().getAsLong(); LOG.debug("Removing data corresponding to log index {} min index {} " + "from cache", index, minIndex); - stateMachineDataCache.removeIf(k -> k <= (Math.min(minIndex, index))); + removeCacheDataUpTo(Math.min(minIndex, index)); } } catch (Exception e) { throw new RuntimeException(e); @@ -984,10 +984,14 @@ private void removeStateMachineDataIfMajorityFollowSync(long index) { if (!waitOnBothFollowers) { // if majority follow in sync, remove all cache previous to current index // including current index - stateMachineDataCache.removeIf(k -> k <= index); + removeCacheDataUpTo(index); } } + private void removeCacheDataUpTo(long index) { + stateMachineDataCache.removeIf(k -> k <= index); + } + private static CompletableFuture completeExceptionally(Exception e) { final CompletableFuture future = new CompletableFuture<>(); future.completeExceptionally(e);