From 30805f6bf871ef4ffad5fba71f685f419f6189fe Mon Sep 17 00:00:00 2001 From: weizuo Date: Tue, 24 Aug 2021 21:57:14 +0800 Subject: [PATCH 1/5] select src replica with min version count for clone task --- .../org/apache/doris/clone/TabletSchedCtx.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index fd3668938cc3b2..4aafbe8bc0b27d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -489,7 +489,9 @@ public void chooseSrcReplica(Map backendsWorkingSlots) throws Sc if (candidates.isEmpty()) { throw new SchedException(Status.UNRECOVERABLE, "unable to find source replica"); } - + + long minVersionCount = -1; + boolean findSrcReplica = false; // choose a replica which slot is available from candidates. for (Replica srcReplica : candidates) { PathSlot slot = backendsWorkingSlots.get(srcReplica.getBackendId()); @@ -499,10 +501,17 @@ public void chooseSrcReplica(Map backendsWorkingSlots) throws Sc long srcPathHash = slot.takeSlot(srcReplica.getPathHash()); if (srcPathHash != -1) { - setSrc(srcReplica); - return; + findSrcReplica = true; + if (minVersionCount == -1 || srcReplica.getVersionCount() < minVersionCount) { + minVersionCount = srcReplica.getVersionCount(); + setSrc(srcReplica); + } } } + + if (findSrcReplica) { + return; + } throw new SchedException(Status.SCHEDULE_FAILED, "unable to find source slot"); } From 3d91e84e54b83a0017d110ee7c77fb16cc756741 Mon Sep 17 00:00:00 2001 From: weizuo Date: Thu, 26 Aug 2021 11:29:05 +0800 Subject: [PATCH 2/5] select src replica with min version count for clone task --- .../src/main/java/org/apache/doris/clone/TabletSchedCtx.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index 4aafbe8bc0b27d..eb764b53dad9c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -501,10 +501,10 @@ public void chooseSrcReplica(Map backendsWorkingSlots) throws Sc long srcPathHash = slot.takeSlot(srcReplica.getPathHash()); if (srcPathHash != -1) { - findSrcReplica = true; if (minVersionCount == -1 || srcReplica.getVersionCount() < minVersionCount) { minVersionCount = srcReplica.getVersionCount(); setSrc(srcReplica); + findSrcReplica = true; } } } From 4410d81428c233dbbcc055be9f330a4c98ff5ad6 Mon Sep 17 00:00:00 2001 From: weizuo Date: Fri, 27 Aug 2021 12:51:36 +0800 Subject: [PATCH 3/5] select src replica with min version count for clone task --- .../org/apache/doris/clone/TabletSchedCtx.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index eb764b53dad9c5..e7d0f03d9731f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -490,7 +490,7 @@ public void chooseSrcReplica(Map backendsWorkingSlots) throws Sc throw new SchedException(Status.UNRECOVERABLE, "unable to find source replica"); } - long minVersionCount = -1; + long minVersionCount = -2; boolean findSrcReplica = false; // choose a replica which slot is available from candidates. for (Replica srcReplica : candidates) { @@ -501,10 +501,20 @@ public void chooseSrcReplica(Map backendsWorkingSlots) throws Sc long srcPathHash = slot.takeSlot(srcReplica.getPathHash()); if (srcPathHash != -1) { - if (minVersionCount == -1 || srcReplica.getVersionCount() < minVersionCount) { + findSrcReplica = true; + if (minVersionCount == -2) { minVersionCount = srcReplica.getVersionCount(); setSrc(srcReplica); - findSrcReplica = true; + } else if (minVersionCount == -1) { + if (srcReplica.getVersionCount() != -1) { + minVersionCount = srcReplica.getVersionCount(); + setSrc(srcReplica); + } + } else { + if (srcReplica.getVersionCount() != -1 && srcReplica.getVersionCount() < minVersionCount) { + minVersionCount = srcReplica.getVersionCount(); + setSrc(srcReplica); + } } } } From d6239f050a624320d6d1312171b2de6003947462 Mon Sep 17 00:00:00 2001 From: weizuo Date: Fri, 27 Aug 2021 13:39:22 +0800 Subject: [PATCH 4/5] select src replica with min version count for clone task --- .../apache/doris/clone/TabletSchedCtx.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index e7d0f03d9731f1..9d1a7ca445e49c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -490,9 +490,9 @@ public void chooseSrcReplica(Map backendsWorkingSlots) throws Sc throw new SchedException(Status.UNRECOVERABLE, "unable to find source replica"); } - long minVersionCount = -2; - boolean findSrcReplica = false; // choose a replica which slot is available from candidates. + long minVersionCount = Long.MAX_VALUE; + boolean findSrcReplica = false; for (Replica srcReplica : candidates) { PathSlot slot = backendsWorkingSlots.get(srcReplica.getBackendId()); if (slot == null) { @@ -501,19 +501,18 @@ public void chooseSrcReplica(Map backendsWorkingSlots) throws Sc long srcPathHash = slot.takeSlot(srcReplica.getPathHash()); if (srcPathHash != -1) { - findSrcReplica = true; - if (minVersionCount == -2) { - minVersionCount = srcReplica.getVersionCount(); + if (!findSrcReplica) { + minVersionCount = srcReplica.getVersionCount() == -1 ? Long.MAX_VALUE : srcReplica.getVersionCount(); setSrc(srcReplica); - } else if (minVersionCount == -1) { - if (srcReplica.getVersionCount() != -1) { - minVersionCount = srcReplica.getVersionCount(); - setSrc(srcReplica); - } + findSrcReplica = true; } else { - if (srcReplica.getVersionCount() != -1 && srcReplica.getVersionCount() < minVersionCount) { - minVersionCount = srcReplica.getVersionCount(); + // version count is set by report process, so it may not be set yet and default value is -1. + // so we need to check it. + long curVerCount = srcReplica.getVersionCount() == -1 ? Long.MAX_VALUE : srcReplica.getVersionCount(); + if (curVerCount < minVersionCount) { + minVersionCount = curVerCount; setSrc(srcReplica); + findSrcReplica = true; } } } From 4dddb050cf666d074434979cd4bdc015ecfe3e58 Mon Sep 17 00:00:00 2001 From: weizuo Date: Fri, 27 Aug 2021 13:47:23 +0800 Subject: [PATCH 5/5] select src replica with min version count for clone task --- .../src/main/java/org/apache/doris/clone/TabletSchedCtx.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index 9d1a7ca445e49c..8e3ab510660b23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -502,12 +502,12 @@ public void chooseSrcReplica(Map backendsWorkingSlots) throws Sc long srcPathHash = slot.takeSlot(srcReplica.getPathHash()); if (srcPathHash != -1) { if (!findSrcReplica) { + // version count is set by report process, so it may not be set yet and default value is -1. + // so we need to check it. minVersionCount = srcReplica.getVersionCount() == -1 ? Long.MAX_VALUE : srcReplica.getVersionCount(); setSrc(srcReplica); findSrcReplica = true; } else { - // version count is set by report process, so it may not be set yet and default value is -1. - // so we need to check it. long curVerCount = srcReplica.getVersionCount() == -1 ? Long.MAX_VALUE : srcReplica.getVersionCount(); if (curVerCount < minVersionCount) { minVersionCount = curVerCount;