From 12456d76d445df98e6e7853b3a90bf62b5398f26 Mon Sep 17 00:00:00 2001 From: HouliangQi Date: Tue, 24 Oct 2023 11:57:37 +0800 Subject: [PATCH 1/6] fix the tablet will be deleted when clone due to concurrency --- .../src/main/java/org/apache/doris/master/MasterImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java index 64b771663b2465..34f6292a60c5ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java @@ -510,7 +510,11 @@ private void finishClone(AgentTask task, TFinishTaskRequest request) { } else { LOG.warn("invalid clone task, ignore it. {}", task); } - + if (request.isSetReportVersion()) { + long reportVersion = request.getReportVersion(); + Env.getCurrentSystemInfo().updateBackendReportVersion( + task.getBackendId(), reportVersion, task.getDbId(), task.getTableId()); + } AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.CLONE, task.getSignature()); } From cbddcd8392ad1a0f5759bb44cc1e94359401ecbc Mon Sep 17 00:00:00 2001 From: HouliangQi Date: Tue, 24 Oct 2023 12:00:08 +0800 Subject: [PATCH 2/6] fix the tablet will be deleted when clone due to concurrency --- be/src/agent/task_worker_pool.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index ff22f7923c3dd6..c9a389d67ddf80 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1900,6 +1900,7 @@ void CloneTaskPool::_clone_worker_thread_callback() { finish_task_request.__set_task_type(agent_task_req.task_type); finish_task_request.__set_signature(agent_task_req.signature); finish_task_request.__set_task_status(status.to_thrift()); + finish_task_request.__set_report_version(_s_report_version); if (!status.ok()) { DorisMetrics::instance()->clone_requests_failed->increment(1); From 54ebe03e375b7e5071fac281b290493345081a09 Mon Sep 17 00:00:00 2001 From: HouliangQi Date: Tue, 24 Oct 2023 12:06:17 +0800 Subject: [PATCH 3/6] fix the tablet will be deleted when clone due to concurrency --- .../main/java/org/apache/doris/master/ReportHandler.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index efca6c7f950a01..0a7ae7691a2822 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -784,6 +784,10 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta continue; } + BinlogConfig binlogConfig = new BinlogConfig(olapTable.getBinlogConfig()); + + ReplicaState state = replica.getState(); + // check report version again long currentBackendReportVersion = Env.getCurrentSystemInfo() .getBackendReportVersion(backendId); @@ -791,9 +795,6 @@ private static void deleteFromMeta(ListMultimap tabletDeleteFromMeta continue; } - BinlogConfig binlogConfig = new BinlogConfig(olapTable.getBinlogConfig()); - - ReplicaState state = replica.getState(); if (state == ReplicaState.NORMAL || state == ReplicaState.SCHEMA_CHANGE) { // if state is PENDING / ROLLUP / CLONE // it's normal that the replica is not created in BE but exists in meta. From c178d74caaec304fe2f138a113ae472f02a45d7f Mon Sep 17 00:00:00 2001 From: HouliangQi Date: Wed, 25 Oct 2023 23:30:21 +0800 Subject: [PATCH 4/6] fix the tablet will be deleted when clone due to concurrency --- be/src/agent/task_worker_pool.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index c9a389d67ddf80..787292c09394fb 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1900,8 +1900,7 @@ void CloneTaskPool::_clone_worker_thread_callback() { finish_task_request.__set_task_type(agent_task_req.task_type); finish_task_request.__set_signature(agent_task_req.signature); finish_task_request.__set_task_status(status.to_thrift()); - finish_task_request.__set_report_version(_s_report_version); - + if (!status.ok()) { DorisMetrics::instance()->clone_requests_failed->increment(1); LOG_WARNING("failed to clone tablet") @@ -1912,9 +1911,12 @@ void CloneTaskPool::_clone_worker_thread_callback() { LOG_INFO("successfully clone tablet") .tag("signature", agent_task_req.signature) .tag("tablet_id", clone_req.tablet_id); + ++_s_report_version; finish_task_request.__set_finish_tablet_infos(tablet_infos); } + finish_task_request.__set_report_version(_s_report_version); + _finish_task(finish_task_request); _remove_task_info(agent_task_req.task_type, agent_task_req.signature); } From 5837ddf45a1825281f60323549b48ac23f06d81b Mon Sep 17 00:00:00 2001 From: HouliangQi Date: Wed, 25 Oct 2023 23:31:10 +0800 Subject: [PATCH 5/6] fix the tablet will be deleted when clone due to concurrency --- be/src/agent/task_worker_pool.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 787292c09394fb..5f71eb9f6e7e16 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1900,7 +1900,6 @@ void CloneTaskPool::_clone_worker_thread_callback() { finish_task_request.__set_task_type(agent_task_req.task_type); finish_task_request.__set_signature(agent_task_req.signature); finish_task_request.__set_task_status(status.to_thrift()); - if (!status.ok()) { DorisMetrics::instance()->clone_requests_failed->increment(1); LOG_WARNING("failed to clone tablet") @@ -1914,9 +1913,7 @@ void CloneTaskPool::_clone_worker_thread_callback() { ++_s_report_version; finish_task_request.__set_finish_tablet_infos(tablet_infos); } - finish_task_request.__set_report_version(_s_report_version); - _finish_task(finish_task_request); _remove_task_info(agent_task_req.task_type, agent_task_req.signature); } From 5bbc441b2a778d24db7ef0c01b616fc6e4403fe0 Mon Sep 17 00:00:00 2001 From: HouliangQi Date: Wed, 25 Oct 2023 23:32:48 +0800 Subject: [PATCH 6/6] fix the tablet will be deleted when clone due to concurrency --- be/src/agent/task_worker_pool.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 5f71eb9f6e7e16..4540d5e04014aa 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1900,6 +1900,7 @@ void CloneTaskPool::_clone_worker_thread_callback() { finish_task_request.__set_task_type(agent_task_req.task_type); finish_task_request.__set_signature(agent_task_req.signature); finish_task_request.__set_task_status(status.to_thrift()); + if (!status.ok()) { DorisMetrics::instance()->clone_requests_failed->increment(1); LOG_WARNING("failed to clone tablet")