From f6956b7c8bbd06983944a0c13b459b5fc6ee90a2 Mon Sep 17 00:00:00 2001 From: zhannngchen <48427519+zhannngchen@users.noreply.github.com> Date: Mon, 11 Nov 2024 20:28:23 +0800 Subject: [PATCH 1/2] [fix](compaction) reduce memory cost for cloud compaction of mow table (#43502) Related PR: #36865 Problem Summary: But when we merge the codes for cloud, such optimization is not applied for cloud compaction We found several cases that compaction of MoW table consume lots of memory on cloud, this PR try to fix this issue Co-authored-by: Chen Zhang --- be/src/cloud/cloud_tablet.cpp | 74 +++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 71030e3bf26e18..3029a55807a0ed 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -762,37 +762,51 @@ Status CloudTablet::calc_delete_bitmap_for_compaction( int64_t initiator, DeleteBitmapPtr& output_rowset_delete_bitmap, bool allow_delete_in_cumu_compaction) { output_rowset_delete_bitmap = std::make_shared(tablet_id()); - std::set missed_rows; - std::map>> location_map; + std::unique_ptr missed_rows; + if ((config::enable_missing_rows_correctness_check || + config::enable_mow_compaction_correctness_check_core) && + !allow_delete_in_cumu_compaction && + compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION) { + missed_rows = std::make_unique(); + LOG(INFO) << "RowLocation Set inited succ for tablet:" << tablet_id(); + } + + std::unique_ptr> location_map; + if (config::enable_rowid_conversion_correctness_check) { + location_map = std::make_unique>(); + LOG(INFO) << "Location Map inited succ for tablet:" << tablet_id(); + } // 1. calc delete bitmap for historical data RETURN_IF_ERROR(_engine.meta_mgr().sync_tablet_rowsets(this)); Version version = max_version(); + std::size_t missed_rows_size = 0; calc_compaction_output_rowset_delete_bitmap( - input_rowsets, rowid_conversion, 0, version.second + 1, &missed_rows, &location_map, - tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get()); - std::size_t missed_rows_size = missed_rows.size(); - if (!allow_delete_in_cumu_compaction) { - if (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION && - tablet_state() == TABLET_RUNNING) { - if (merged_rows >= 0 && merged_rows != missed_rows_size) { - std::string err_msg = fmt::format( - "cumulative compaction: the merged rows({}) is not equal to missed " - "rows({}) in rowid conversion, tablet_id: {}, table_id:{}", - merged_rows, missed_rows_size, tablet_id(), table_id()); - if (config::enable_mow_compaction_correctness_check_core) { - CHECK(false) << err_msg; - } else { - DCHECK(false) << err_msg; + input_rowsets, rowid_conversion, 0, version.second + 1, missed_rows.get(), + location_map.get(), tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get()); + if (missed_rows) { + std::size_t missed_rows_size = missed_rows.size(); + if (!allow_delete_in_cumu_compaction) { + if (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION && + tablet_state() == TABLET_RUNNING) { + if (merged_rows >= 0 && merged_rows != missed_rows_size) { + std::string err_msg = fmt::format( + "cumulative compaction: the merged rows({}) is not equal to missed " + "rows({}) in rowid conversion, tablet_id: {}, table_id:{}", + merged_rows, missed_rows_size, tablet_id(), table_id()); + if (config::enable_mow_compaction_correctness_check_core) { + CHECK(false) << err_msg; + } else { + DCHECK(false) << err_msg; + } } - LOG(WARNING) << err_msg; } } } - if (config::enable_rowid_conversion_correctness_check) { - RETURN_IF_ERROR(check_rowid_conversion(output_rowset, location_map)); + if (location_map) { + RETURN_IF_ERROR(check_rowid_conversion(output_rowset, *location_map)); + location_map->clear(); } - location_map.clear(); // 2. calc delete bitmap for incremental data RETURN_IF_ERROR(_engine.meta_mgr().get_delete_bitmap_update_lock( @@ -800,16 +814,16 @@ Status CloudTablet::calc_delete_bitmap_for_compaction( RETURN_IF_ERROR(_engine.meta_mgr().sync_tablet_rowsets(this)); calc_compaction_output_rowset_delete_bitmap( - input_rowsets, rowid_conversion, version.second, UINT64_MAX, &missed_rows, - &location_map, tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get()); - if (config::enable_rowid_conversion_correctness_check) { - RETURN_IF_ERROR(check_rowid_conversion(output_rowset, location_map)); - } - if (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION) { - DCHECK_EQ(missed_rows.size(), missed_rows_size); - if (missed_rows.size() != missed_rows_size) { + input_rowsets, rowid_conversion, version.second, UINT64_MAX, missed_rows.get(), + location_map.get(), tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get()); + if (location_map) { + RETURN_IF_ERROR(check_rowid_conversion(output_rowset, *location_map)); + } + if (missed_rows) { + DCHECK_EQ(missed_rows->size(), missed_rows_size); + if (missed_rows->size() != missed_rows_size) { LOG(WARNING) << "missed rows don't match, before: " << missed_rows_size - << " after: " << missed_rows.size(); + << " after: " << missed_rows->size(); } } From cee0cf7fe1701a021db6674c2366d20faa8960ea Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Mon, 11 Nov 2024 21:48:46 +0800 Subject: [PATCH 2/2] fix compile --- be/src/cloud/cloud_tablet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 3029a55807a0ed..f50ffd555e0161 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -785,7 +785,7 @@ Status CloudTablet::calc_delete_bitmap_for_compaction( input_rowsets, rowid_conversion, 0, version.second + 1, missed_rows.get(), location_map.get(), tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get()); if (missed_rows) { - std::size_t missed_rows_size = missed_rows.size(); + std::size_t missed_rows_size = missed_rows->size(); if (!allow_delete_in_cumu_compaction) { if (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION && tablet_state() == TABLET_RUNNING) {