From f565e9574f9080e38e509af49e9b3c0aef40d339 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 25 Nov 2025 15:25:16 -0300 Subject: [PATCH 1/3] cleanup export manifests at shutdown --- src/Storages/MergeTree/MergeTreeData.cpp | 15 ++------------- .../MergeTree/MergeTreePartExportManifest.h | 1 - src/Storages/StorageMergeTree.cpp | 5 +++++ src/Storages/StorageReplicatedMergeTree.cpp | 11 +++++++++++ 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6f1c59239b93..17f4ad131379 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6283,14 +6283,7 @@ void MergeTreeData::killExportPart(const String & transaction_id) std::erase_if(export_manifests, [&](const auto & manifest) { - if (manifest.transaction_id == transaction_id) - { - if (manifest.task) - manifest.task->cancel(); - - return true; - } - return false; + return manifest.transaction_id == transaction_id; }); } @@ -9137,17 +9130,13 @@ bool MergeTreeData::scheduleDataMovingJob(BackgroundJobsAssignee & assignee) context_copy->setCurrentQueryId(manifest.transaction_id); context_copy->setBackgroundOperationTypeForContext(ClientInfo::BackgroundOperationType::EXPORT_PART); - auto task = std::make_shared(*this, manifest, context_copy); - - manifest.in_progress = assignee.scheduleMoveTask(task); + manifest.in_progress = assignee.scheduleMoveTask(std::make_shared(*this, manifest, context_copy)); if (!manifest.in_progress) { continue; } - manifest.task = task; - return true; } diff --git a/src/Storages/MergeTree/MergeTreePartExportManifest.h b/src/Storages/MergeTree/MergeTreePartExportManifest.h index 533eeb6decdd..e99a3fff0cf7 100644 --- a/src/Storages/MergeTree/MergeTreePartExportManifest.h +++ b/src/Storages/MergeTree/MergeTreePartExportManifest.h @@ -73,7 +73,6 @@ struct MergeTreePartExportManifest time_t create_time; mutable bool in_progress = false; - mutable std::shared_ptr task = nullptr; bool operator<(const MergeTreePartExportManifest & rhs) const { diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index a46b85364be0..1cc616bd8521 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -256,6 +256,11 @@ void StorageMergeTree::shutdown(bool) if (deduplication_log) deduplication_log->shutdown(); + + { + std::lock_guard lock(export_manifests_mutex); + export_manifests.clear(); + } } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 0b7c3c600e5f..79cd5830508d 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -6014,6 +6014,17 @@ void StorageReplicatedMergeTree::shutdown(bool) /// Wait for all of them std::lock_guard lock(data_parts_exchange_ptr->rwlock); } + + { + std::lock_guard lock(export_manifests_mutex); + export_manifests.clear(); + } + + { + std::lock_guard lock(export_merge_tree_partition_mutex); + export_merge_tree_partition_task_entries.clear(); + } + LOG_TRACE(log, "Shutdown finished"); } From 6cec57006734eb36cb89b8e58fdf14cea2240c5b Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 25 Nov 2025 15:32:06 -0300 Subject: [PATCH 2/3] reorder --- src/Storages/StorageReplicatedMergeTree.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 79cd5830508d..ec6c43ff775e 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -6016,13 +6016,13 @@ void StorageReplicatedMergeTree::shutdown(bool) } { - std::lock_guard lock(export_manifests_mutex); - export_manifests.clear(); + std::lock_guard lock(export_merge_tree_partition_mutex); + export_merge_tree_partition_task_entries.clear(); } { - std::lock_guard lock(export_merge_tree_partition_mutex); - export_merge_tree_partition_task_entries.clear(); + std::lock_guard lock(export_manifests_mutex); + export_manifests.clear(); } LOG_TRACE(log, "Shutdown finished"); From 6a99acf7ab63fd8814b5b3a900da0b700ba8d7d4 Mon Sep 17 00:00:00 2001 From: Arthur Passos Date: Tue, 25 Nov 2025 15:43:53 -0300 Subject: [PATCH 3/3] I am just too crazy at this point --- src/Storages/MergeTree/MergeTreeData.cpp | 15 +++++++++++++-- .../MergeTree/MergeTreePartExportManifest.h | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 17f4ad131379..6f1c59239b93 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -6283,7 +6283,14 @@ void MergeTreeData::killExportPart(const String & transaction_id) std::erase_if(export_manifests, [&](const auto & manifest) { - return manifest.transaction_id == transaction_id; + if (manifest.transaction_id == transaction_id) + { + if (manifest.task) + manifest.task->cancel(); + + return true; + } + return false; }); } @@ -9130,13 +9137,17 @@ bool MergeTreeData::scheduleDataMovingJob(BackgroundJobsAssignee & assignee) context_copy->setCurrentQueryId(manifest.transaction_id); context_copy->setBackgroundOperationTypeForContext(ClientInfo::BackgroundOperationType::EXPORT_PART); - manifest.in_progress = assignee.scheduleMoveTask(std::make_shared(*this, manifest, context_copy)); + auto task = std::make_shared(*this, manifest, context_copy); + + manifest.in_progress = assignee.scheduleMoveTask(task); if (!manifest.in_progress) { continue; } + manifest.task = task; + return true; } diff --git a/src/Storages/MergeTree/MergeTreePartExportManifest.h b/src/Storages/MergeTree/MergeTreePartExportManifest.h index e99a3fff0cf7..533eeb6decdd 100644 --- a/src/Storages/MergeTree/MergeTreePartExportManifest.h +++ b/src/Storages/MergeTree/MergeTreePartExportManifest.h @@ -73,6 +73,7 @@ struct MergeTreePartExportManifest time_t create_time; mutable bool in_progress = false; + mutable std::shared_ptr task = nullptr; bool operator<(const MergeTreePartExportManifest & rhs) const {