From c9385b4914caaa0b6acb22841b8c11fda82892ba Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 8 Nov 2024 21:41:06 +0800 Subject: [PATCH] [fix](load) fix broker load progress due to retry (#42959) ## Proposed changes Currently, when retrying a broker load, it will use a different `load_id` with the same `job_id`. The `total_scan_nums` in progress is accumulated by `job_id`. This will cause the `total_scan_nums` progress to be multiple of the actual scan nums. For example, suppose a 10 instance broker load gets retried: ``` initial (0/10) 0% -> retry (0/20) 0% -> finish (10/20) 50% ``` --- .../java/org/apache/doris/load/loadv2/LoadLoadingTask.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java index 9b74f8718af1da..8510c2dc124be8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.BrokerDesc; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EnvFactory; import org.apache.doris.catalog.OlapTable; import org.apache.doris.common.Config; @@ -217,6 +218,8 @@ public void updateRetryInfo() { UUID uuid = UUID.randomUUID(); this.loadId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); planner.updateLoadId(this.loadId); + // reset progress on each retry, otherwise the finished/total num will be incorrect + Env.getCurrentProgressManager().registerProgressSimple(String.valueOf(callback.getCallbackId())); } void settWorkloadGroups(List tWorkloadGroups) {