From 2c4d8685f48e7fc897d6aafd2b6af02463893c00 Mon Sep 17 00:00:00 2001 From: meiyi Date: Wed, 4 Jun 2025 15:41:57 +0800 Subject: [PATCH] [fix](mow) remove enable_delete_bitmap_merge_on_compaction --- be/src/cloud/cloud_cumulative_compaction.cpp | 47 --- be/src/cloud/cloud_cumulative_compaction.h | 2 - be/src/cloud/cloud_meta_mgr.cpp | 18 - be/src/cloud/cloud_meta_mgr.h | 4 - be/src/cloud/cloud_tablet.cpp | 3 - be/src/common/config.cpp | 1 - be/src/common/config.h | 1 - be/src/olap/compaction.cpp | 64 --- be/src/olap/compaction.h | 8 - be/src/olap/tablet.cpp | 3 - be/src/olap/tablet_meta.cpp | 48 --- be/src/olap/tablet_meta.h | 10 - ...ction_remove_old_version_delete_bitmap.out | 78 ---- .../test_mow_compaction_and_schema_change.out | 37 -- ...on_remove_old_version_delete_bitmap.groovy | 383 ------------------ ...st_mow_compaction_and_schema_change.groovy | 11 +- 16 files changed, 1 insertion(+), 717 deletions(-) delete mode 100644 regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out delete mode 100644 regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index d1f78e96c42e23..f00fc609c1be52 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -389,11 +389,6 @@ Status CloudCumulativeCompaction::modify_rowsets() { stats.num_rows(), stats.data_size()); } } - if (config::enable_delete_bitmap_merge_on_compaction && - _tablet->keys_type() == KeysType::UNIQUE_KEYS && - _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) { - RETURN_IF_ERROR(process_old_version_delete_bitmap()); - } // agg delete bitmap for pre rowsets if (config::enable_agg_and_remove_pre_rowsets_delete_bitmap && _tablet->keys_type() == KeysType::UNIQUE_KEYS && @@ -440,48 +435,6 @@ Status CloudCumulativeCompaction::modify_rowsets() { return Status::OK(); } -Status CloudCumulativeCompaction::process_old_version_delete_bitmap() { - // agg previously rowset old version delete bitmap - std::vector pre_rowsets {}; - for (const auto& it : cloud_tablet()->rowset_map()) { - if (it.first.second < _input_rowsets.front()->start_version()) { - pre_rowsets.emplace_back(it.second); - } - } - std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator); - if (!pre_rowsets.empty()) { - std::vector> - to_remove_vec; - DeleteBitmapPtr new_delete_bitmap = nullptr; - agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap); - if (!new_delete_bitmap->empty()) { - // store agg delete bitmap - DBUG_EXECUTE_IF("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed", - { - return Status::InternalError( - "test fail to update delete bitmap for tablet_id {}", - cloud_tablet()->tablet_id()); - }); - std::map rowset_to_versions; - RETURN_IF_ERROR(_engine.meta_mgr().cloud_update_delete_bitmap_without_lock( - *cloud_tablet(), new_delete_bitmap.get(), rowset_to_versions)); - - Version version(_input_rowsets.front()->start_version(), - _input_rowsets.back()->end_version()); - for (auto it = new_delete_bitmap->delete_bitmap.begin(); - it != new_delete_bitmap->delete_bitmap.end(); it++) { - _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second); - } - _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), - to_remove_vec); - DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", { - static_cast(_tablet.get())->delete_expired_stale_rowsets(); - }); - } - } - return Status::OK(); -} - Status CloudCumulativeCompaction::garbage_collection() { RETURN_IF_ERROR(CloudCompactionMixin::garbage_collection()); cloud::TabletJobInfoPB job; diff --git a/be/src/cloud/cloud_cumulative_compaction.h b/be/src/cloud/cloud_cumulative_compaction.h index ba3b768a24ff10..e139054bbc7ee8 100644 --- a/be/src/cloud/cloud_cumulative_compaction.h +++ b/be/src/cloud/cloud_cumulative_compaction.h @@ -52,8 +52,6 @@ class CloudCumulativeCompaction : public CloudCompactionMixin { void update_cumulative_point(); - Status process_old_version_delete_bitmap(); - ReaderType compaction_type() const override { return ReaderType::READER_CUMULATIVE_COMPACTION; } int64_t _input_segments = 0; diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 40a568b3d143b6..73c9d0d72a8abf 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -1434,24 +1434,6 @@ void CloudMetaMgr::remove_delete_bitmap_update_lock(int64_t table_id, int64_t lo } } -Status CloudMetaMgr::remove_old_version_delete_bitmap( - int64_t tablet_id, - const std::vector>& to_delete) { - LOG(INFO) << "remove_old_version_delete_bitmap , tablet_id: " << tablet_id; - RemoveDeleteBitmapRequest req; - RemoveDeleteBitmapResponse res; - req.set_cloud_unique_id(config::cloud_unique_id); - req.set_tablet_id(tablet_id); - for (auto& value : to_delete) { - req.add_rowset_ids(std::get<0>(value)); - req.add_begin_versions(std::get<1>(value)); - req.add_end_versions(std::get<2>(value)); - } - auto st = retry_rpc("remove old delete bitmap", req, &res, - &MetaService_Stub::remove_delete_bitmap); - return st; -} - void CloudMetaMgr::check_table_size_correctness(const RowsetMeta& rs_meta) { if (!config::enable_table_size_correctness_check) { return; diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h index a40ec7e816d58c..421e0b28bf3eca 100644 --- a/be/src/cloud/cloud_meta_mgr.h +++ b/be/src/cloud/cloud_meta_mgr.h @@ -117,10 +117,6 @@ class CloudMetaMgr { void remove_delete_bitmap_update_lock(int64_t table_id, int64_t lock_id, int64_t initiator, int64_t tablet_id); - Status remove_old_version_delete_bitmap( - int64_t tablet_id, - const std::vector>& to_delete); - private: bool sync_tablet_delete_bitmap_by_cache(CloudTablet* tablet, int64_t old_max_version, std::ranges::range auto&& rs_metas, diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index f0ee3a8c5e3fa9..68d23272f05ab0 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -397,7 +397,6 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() { std::vector>> deleted_stale_rowsets; int64_t expired_stale_sweep_endtime = ::time(nullptr) - config::tablet_rowset_stale_sweep_time_sec; - std::vector version_to_delete; { std::unique_lock wlock(_meta_lock); @@ -438,14 +437,12 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() { _tablet_meta->delete_stale_rs_meta_by_version(v_ts->version()); } Version version(start_version, end_version); - version_to_delete.emplace_back(version.to_string()); if (!stale_rowsets.empty()) { deleted_stale_rowsets.emplace_back(version, std::move(stale_rowsets)); } } _reconstruct_version_tracker_if_necessary(); } - _tablet_meta->delete_bitmap().remove_stale_delete_bitmap_from_queue(version_to_delete); recycle_cached_data(expired_rowsets); if (config::enable_mow_verbose_log) { LOG_INFO("finish delete_expired_stale_rowset for tablet={}", tablet_id()); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 43936fc4e76eaf..f262da0364a27d 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1472,7 +1472,6 @@ DEFINE_mInt32(check_score_rounds_num, "1000"); DEFINE_Int32(query_cache_size, "512"); -DEFINE_mBool(enable_delete_bitmap_merge_on_compaction, "false"); // Enable validation to check the correctness of table size. DEFINE_Bool(enable_table_size_correctness_check, "false"); DEFINE_Bool(force_regenerate_rowsetid_on_start_error, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 1c1c64ea811a1f..b2058f6c2db598 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1546,7 +1546,6 @@ DECLARE_mInt32(check_score_rounds_num); DECLARE_Int32(query_cache_size); DECLARE_Bool(force_regenerate_rowsetid_on_start_error); -DECLARE_mBool(enable_delete_bitmap_merge_on_compaction); // Enable validation to check the correctness of table size. DECLARE_Bool(enable_table_size_correctness_check); // Enable sleep 5s between delete cumulative compaction. diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index ddfab2cf648aef..13616cefc1a9f2 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -1049,33 +1049,6 @@ Status CloudCompactionMixin::update_delete_bitmap() { return Status::OK(); } -void Compaction::agg_and_remove_old_version_delete_bitmap( - std::vector& pre_rowsets, - std::vector>& - to_remove_vec, - DeleteBitmapPtr& new_delete_bitmap) { - // agg previously rowset old version delete bitmap - auto pre_max_version = _output_rowset->version().second; - new_delete_bitmap = std::make_shared(_tablet->tablet_meta()->tablet_id()); - for (auto& rowset : pre_rowsets) { - if (rowset->rowset_meta()->total_disk_size() == 0) { - continue; - } - for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) { - rowset->rowset_id().to_string(); - DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0}; - DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version}; - auto d = _tablet->tablet_meta()->delete_bitmap().get_agg( - {rowset->rowset_id(), seg_id, pre_max_version}); - to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end)); - if (d->isEmpty()) { - continue; - } - new_delete_bitmap->set(end, *d); - } - } -} - Status CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx) { // only do index compaction for dup_keys and unique_keys with mow enabled if (config::inverted_index_compaction_enable && @@ -1277,13 +1250,6 @@ Status CompactionMixin::modify_rowsets() { tablet()->delete_expired_stale_rowset(); } - if (config::enable_delete_bitmap_merge_on_compaction && - compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION && - _tablet->keys_type() == KeysType::UNIQUE_KEYS && - _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size() != 1) { - process_old_version_delete_bitmap(); - } - int64_t cur_max_version = 0; { std::shared_lock rlock(_tablet->get_header_lock()); @@ -1303,36 +1269,6 @@ Status CompactionMixin::modify_rowsets() { return Status::OK(); } -void CompactionMixin::process_old_version_delete_bitmap() { - std::vector pre_rowsets {}; - for (const auto& it : tablet()->rowset_map()) { - if (it.first.second < _input_rowsets.front()->start_version()) { - pre_rowsets.emplace_back(it.second); - } - } - std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator); - if (!pre_rowsets.empty()) { - std::vector> - to_remove_vec; - DeleteBitmapPtr new_delete_bitmap = nullptr; - agg_and_remove_old_version_delete_bitmap(pre_rowsets, to_remove_vec, new_delete_bitmap); - if (!new_delete_bitmap->empty()) { - // store agg delete bitmap - Version version(_input_rowsets.front()->start_version(), - _input_rowsets.back()->end_version()); - for (auto it = new_delete_bitmap->delete_bitmap.begin(); - it != new_delete_bitmap->delete_bitmap.end(); it++) { - _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second); - } - _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), - to_remove_vec); - DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", { - static_cast(_tablet.get())->delete_expired_stale_rowset(); - }); - } - } -} - bool CompactionMixin::_check_if_includes_input_rowsets( const RowsetIdUnorderedSet& commit_rowset_ids_set) const { std::vector commit_rowset_ids {}; diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h index a7f83df95bb76a..523acf413dc47a 100644 --- a/be/src/olap/compaction.h +++ b/be/src/olap/compaction.h @@ -92,12 +92,6 @@ class Compaction { virtual Status update_delete_bitmap() = 0; - void agg_and_remove_old_version_delete_bitmap( - std::vector& pre_rowsets, - std::vector>& - to_remove_vec, - DeleteBitmapPtr& new_delete_bitmap); - // the root tracker for this compaction std::shared_ptr _mem_tracker; @@ -180,8 +174,6 @@ class CompactionMixin : public Compaction { Status do_compact_ordered_rowsets(); - void process_old_version_delete_bitmap(); - bool _check_if_includes_input_rowsets(const RowsetIdUnorderedSet& commit_rowset_ids_set) const; void update_compaction_level(); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index a2fcc744366172..5711e080d6fd7d 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -828,7 +828,6 @@ void Tablet::delete_expired_stale_rowset() { auto old_meta_size = _tablet_meta->all_stale_rs_metas().size(); // do delete operation - std::vector version_to_delete; auto to_delete_iter = stale_version_path_map.begin(); while (to_delete_iter != stale_version_path_map.end()) { std::vector& to_delete_version = @@ -867,13 +866,11 @@ void Tablet::delete_expired_stale_rowset() { _delete_stale_rowset_by_version(timestampedVersion->version()); } Version version(start_version, end_version); - version_to_delete.emplace_back(version.to_string()); to_delete_iter++; if (!remove_rowset_ids.empty()) { deleted_stale_rowsets.emplace_back(version, remove_rowset_ids); } } - _tablet_meta->delete_bitmap().remove_stale_delete_bitmap_from_queue(version_to_delete); bool reconstructed = _reconstruct_version_tracker_if_necessary(); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 354cfa85ed3f6c..9f80b4e6fba437 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -1310,54 +1310,6 @@ void DeleteBitmap::merge(const DeleteBitmap& other) { } } -void DeleteBitmap::add_to_remove_queue( - const std::string& version_str, - const std::vector>& - vector) { - std::shared_lock l(stale_delete_bitmap_lock); - _stale_delete_bitmap.emplace(version_str, vector); -} - -void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const std::vector& vector) { - if (!config::enable_delete_bitmap_merge_on_compaction) { - return; - } - std::shared_lock l(stale_delete_bitmap_lock); - // - std::vector> to_delete; - int64_t tablet_id = -1; - for (auto& version_str : vector) { - auto it = _stale_delete_bitmap.find(version_str); - if (it != _stale_delete_bitmap.end()) { - auto delete_bitmap_vector = it->second; - for (auto& delete_bitmap_tuple : it->second) { - if (tablet_id < 0) { - tablet_id = std::get<0>(delete_bitmap_tuple); - } - auto start_bmk = std::get<1>(delete_bitmap_tuple); - auto end_bmk = std::get<2>(delete_bitmap_tuple); - // the key range of to be removed is [start_bmk,end_bmk), - // due to the different definitions of the right boundary, - // so use end_bmk as right boundary when removing local delete bitmap, - // use (end_bmk - 1) as right boundary when removing ms delete bitmap - remove(start_bmk, end_bmk); - to_delete.emplace_back(std::make_tuple(std::get<0>(start_bmk).to_string(), 0, - std::get<2>(end_bmk) - 1)); - } - _stale_delete_bitmap.erase(version_str); - } - } - if (tablet_id == -1 || to_delete.empty() || !config::is_cloud_mode()) { - return; - } - CloudStorageEngine& engine = ExecEnv::GetInstance()->storage_engine().to_cloud(); - auto st = engine.meta_mgr().remove_old_version_delete_bitmap(tablet_id, to_delete); - if (!st.ok()) { - LOG(WARNING) << "fail to remove_stale_delete_bitmap_from_queue for tablet=" << tablet_id - << ",st=" << st; - } -} - uint64_t DeleteBitmap::get_delete_bitmap_count() { std::shared_lock l(lock); uint64_t count = 0; diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index ca9bc11224b98d..7ea31a5f468f97 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -384,7 +384,6 @@ class TabletMeta : public MetadataAdder { class DeleteBitmap { public: mutable std::shared_mutex lock; - mutable std::shared_mutex stale_delete_bitmap_lock; using SegmentId = uint32_t; using Version = uint64_t; using BitmapKey = std::tuple; @@ -551,11 +550,6 @@ class DeleteBitmap { void remove_sentinel_marks(); - void add_to_remove_queue(const std::string& version_str, - const std::vector>& vector); - void remove_stale_delete_bitmap_from_queue(const std::vector& vector); - uint64_t get_delete_bitmap_count(); void traverse_rowset_and_version( @@ -607,10 +601,6 @@ class DeleteBitmap { int64_t _tablet_id; mutable std::shared_mutex _rowset_cache_version_lock; mutable std::map> _rowset_cache_version; - // > - std::map>> - _stale_delete_bitmap; }; inline TabletUid TabletMeta::tablet_uid() const { diff --git a/regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out b/regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out deleted file mode 100644 index 37dfa3b93a5878..00000000000000 --- a/regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out +++ /dev/null @@ -1,78 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !sql -- -0 0 8 -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 6 -7 7 7 -8 8 8 - --- !sql -- -0 0 8 -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 6 -7 7 7 -8 8 8 - --- !sql -- -0 0 13 -1 13 13 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 6 -7 7 7 -8 8 8 - --- !sql -- -0 0 13 -1 13 13 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 6 -7 7 7 -8 8 8 - --- !sql -- -0 0 18 -1 23 23 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 6 -7 7 7 -8 8 8 - --- !sql -- -0 0 18 -1 23 23 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 6 -7 7 7 -8 8 8 - --- !sql -- -0 5 5 -1 28 28 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 6 -7 7 7 -8 8 8 - diff --git a/regression-test/data/compaction/test_mow_compaction_and_schema_change.out b/regression-test/data/compaction/test_mow_compaction_and_schema_change.out index 43ba92316aed59..e912c295a2b382 100644 --- a/regression-test/data/compaction/test_mow_compaction_and_schema_change.out +++ b/regression-test/data/compaction/test_mow_compaction_and_schema_change.out @@ -69,40 +69,3 @@ 4 100 5 100 --- !sql1 -- -1 99 -2 99 -3 99 -4 99 -5 99 - --- !sql2 -- -1 99 -2 99 -3 99 -4 99 -5 99 - --- !sql3 -- -1 100 -2 100 -3 100 -4 100 -5 100 - --- !sql4 -- -1 100 -2 100 -3 100 -4 100 -5 100 - --- !sql5 -- -1 100 -1 99 -2 100 -2 99 -3 100 -4 100 -5 100 - diff --git a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy deleted file mode 100644 index 65ba1818a885a9..00000000000000 --- a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy +++ /dev/null @@ -1,383 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.codehaus.groovy.runtime.IOGroovyMethods - -suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - def backendId_to_params = [string: [:]] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - def set_be_param = { paramName, paramValue -> - // for eache be node, set paramName=paramValue - for (String id in backendId_to_backendIP.keySet()) { - def beIp = backendId_to_backendIP.get(id) - def bePort = backendId_to_backendHttpPort.get(id) - def (code, out, err) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, paramValue)) - assertTrue(out.contains("OK")) - } - } - - def reset_be_param = { paramName -> - // for eache be node, reset paramName to default - for (String id in backendId_to_backendIP.keySet()) { - def beIp = backendId_to_backendIP.get(id) - def bePort = backendId_to_backendHttpPort.get(id) - def original_value = backendId_to_params.get(id).get(paramName) - def (code, out, err) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, original_value)) - assertTrue(out.contains("OK")) - } - } - - def get_be_param = { paramName -> - // for eache be node, get param value by default - def paramValue = "" - for (String id in backendId_to_backendIP.keySet()) { - def beIp = backendId_to_backendIP.get(id) - def bePort = backendId_to_backendHttpPort.get(id) - // get the config value from be - def (code, out, err) = curl("GET", String.format("http://%s:%s/api/show_config?conf_item=%s", beIp, bePort, paramName)) - assertTrue(code == 0) - assertTrue(out.contains(paramName)) - // parsing - def resultList = parseJson(out)[0] - assertTrue(resultList.size() == 4) - // get original value - paramValue = resultList[2] - backendId_to_params.get(id, [:]).put(paramName, paramValue) - } - } - - def triggerCompaction = { be_host, be_http_port, compact_type, tablet_id -> - if (compact_type == "cumulative") { - def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id) - logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) - assertEquals(code_1, 0) - return out_1 - } else if (compact_type == "full") { - def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id) - logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2) - assertEquals(code_2, 0) - return out_2 - } else { - assertFalse(True) - } - } - - def getTabletStatus = { be_host, be_http_port, tablet_id -> - boolean running = true - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://${be_host}:${be_http_port}") - sb.append("/api/compaction/show?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - def process = command.execute() - def code = process.waitFor() - def out = process.getText() - logger.info("Get tablet status: =" + code + ", out=" + out) - assertEquals(code, 0) - def tabletStatus = parseJson(out.trim()) - return tabletStatus - } - - def waitForCompaction = { be_host, be_http_port, tablet_id -> - boolean running = true - do { - Thread.sleep(1000) - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://${be_host}:${be_http_port}") - sb.append("/api/compaction/run_status?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - def process = command.execute() - def code = process.waitFor() - def out = process.getText() - logger.info("Get compaction status: code=" + code + ", out=" + out) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - - def getLocalDeleteBitmapStatus = { be_host, be_http_port, tablet_id -> - boolean running = true - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://${be_host}:${be_http_port}") - sb.append("/api/delete_bitmap/count_local?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - def process = command.execute() - def code = process.waitFor() - def out = process.getText() - logger.info("Get local delete bitmap count status: =" + code + ", out=" + out) - assertEquals(code, 0) - def deleteBitmapStatus = parseJson(out.trim()) - return deleteBitmapStatus - } - - def getMSDeleteBitmapStatus = { be_host, be_http_port, tablet_id -> - boolean running = true - StringBuilder sb = new StringBuilder(); - sb.append("curl -X GET http://${be_host}:${be_http_port}") - sb.append("/api/delete_bitmap/count_ms?tablet_id=") - sb.append(tablet_id) - - String command = sb.toString() - logger.info(command) - def process = command.execute() - def code = process.waitFor() - def out = process.getText() - logger.info("Get ms delete bitmap count status: =" + code + ", out=" + out) - assertEquals(code, 0) - def deleteBitmapStatus = parseJson(out.trim()) - return deleteBitmapStatus - } - - def testTable = "test_cu_compaction_remove_old_version_delete_bitmap" - def timeout = 10000 - sql """ DROP TABLE IF EXISTS ${testTable}""" - def testTableDDL = """ - create table ${testTable} - ( - `plan_id` bigint(20) NOT NULL, - `target_id` int(20) NOT NULL, - `target_name` varchar(255) NOT NULL - ) - ENGINE=OLAP - UNIQUE KEY(`plan_id`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`plan_id`) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "replication_allocation" = "tag.location.default: 1", - "disable_auto_compaction" = "true" - ); - """ - sql testTableDDL - sql "sync" - - // store the original value - get_be_param("compaction_promotion_version_count") - get_be_param("tablet_rowset_stale_sweep_time_sec") - set_be_param("compaction_promotion_version_count", "5") - set_be_param("tablet_rowset_stale_sweep_time_sec", "0") - - try { - GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") - // 1. test normal - sql "sync" - sql """ INSERT INTO ${testTable} VALUES (0,0,'1'),(1,1,'1'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'2'),(2,2,'2'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'3'),(3,3,'3'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'4'),(4,4,'4'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'5'),(5,5,'5'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'6'),(6,6,'6'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'7'),(7,7,'7'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'8'),(8,8,'8'); """ - - qt_sql "select * from ${testTable} order by plan_id" - - // trigger compaction to generate base rowset - def tablets = sql_return_maparray """ show tablets from ${testTable}; """ - logger.info("tablets: " + tablets) - def local_delete_bitmap_count = 0 - def ms_delete_bitmap_count = 0 - def local_delete_bitmap_cardinality = 0; - def ms_delete_bitmap_cardinality = 0; - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - String trigger_backend_id = tablet.BackendId - getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - - // before compaction, delete_bitmap_count is (rowsets num - 1) - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - assertTrue(local_delete_bitmap_count == 7) - assertTrue(local_delete_bitmap_cardinality == 7) - - if (isCloudMode()) { - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) - assertTrue(ms_delete_bitmap_count == 7) - assertTrue(ms_delete_bitmap_cardinality == 7) - } - - - assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - } - - qt_sql "select * from ${testTable} order by plan_id" - - def now = System.currentTimeMillis() - - sql """ INSERT INTO ${testTable} VALUES (0,0,'9'),(1,9,'9'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'10'),(1,10,'10'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'11'),(1,11,'11'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'12'),(1,12,'12'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'13'),(1,13,'13'); """ - - def time_diff = System.currentTimeMillis() - now - logger.info("time_diff:" + time_diff) - assertTrue(time_diff <= timeout, "wait_for_insert_into_values timeout") - - qt_sql "select * from ${testTable} order by plan_id" - - // trigger cu compaction to remove old version delete bitmap - - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - - // before compaction, local delete_bitmap_count is (total rowsets num - 1), ms delete_bitmap_count is new rowset num - String trigger_backend_id = tablet.BackendId - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - assertTrue(local_delete_bitmap_count == 12) - assertTrue(local_delete_bitmap_cardinality == 17) - - if (isCloudMode()) { - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) - assertTrue(ms_delete_bitmap_count == 5) - assertTrue(ms_delete_bitmap_cardinality == 10) - } - - getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - - Thread.sleep(1000) - // after compaction, delete_bitmap_count is 1, cardinality is 2, check it - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - assertTrue(local_delete_bitmap_count == 1) - assertTrue(local_delete_bitmap_cardinality == 2) - if (isCloudMode()) { - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) - assertTrue(ms_delete_bitmap_count == 1) - assertTrue(ms_delete_bitmap_cardinality == 2) - } - } - - qt_sql "select * from ${testTable} order by plan_id" - - // 2. test update delete bitmap failed - - now = System.currentTimeMillis() - - sql """ INSERT INTO ${testTable} VALUES (0,0,'14'),(1,19,'19'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'15'),(1,20,'20'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'16'),(1,21,'21'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'17'),(1,22,'22'); """ - sql """ INSERT INTO ${testTable} VALUES (0,0,'18'),(1,23,'23'); """ - - time_diff = System.currentTimeMillis() - now - logger.info("time_diff:" + time_diff) - assertTrue(time_diff <= timeout, "wait_for_insert_into_values timeout") - - qt_sql "select * from ${testTable} order by plan_id" - GetDebugPoint().enableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed") - if (isCloudMode()) { - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - def tablet_info = sql_return_maparray """ show tablet ${tablet_id}; """ - logger.info("tablet: " + tablet_info) - String trigger_backend_id = tablet.BackendId - - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - ms_delete_bitmap_count = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 6) - assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count) - - local_delete_bitmap_cardinality = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - ms_delete_bitmap_cardinality = getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality - logger.info("local_delete_bitmap_cardinality:" + local_delete_bitmap_cardinality) - logger.info("ms_delete_bitmap_cardinality:" + ms_delete_bitmap_cardinality) - assertTrue(local_delete_bitmap_cardinality == 12) - assertTrue(ms_delete_bitmap_cardinality == 12) - - getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], - "cumulative", tablet_id).contains("Success")); - waitForCompaction(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id) - getTabletStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id); - - // update fail, local delete_bitmap_count will not change - Thread.sleep(1000) - local_delete_bitmap_count = getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id], backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count - logger.info("local_delete_bitmap_count:" + local_delete_bitmap_count) - assertTrue(local_delete_bitmap_count == 6) - } - } - - qt_sql "select * from ${testTable} order by plan_id" - - now = System.currentTimeMillis() - - sql """ INSERT INTO ${testTable} VALUES (0,1,'1'),(1,24,'24'); """ - sql """ INSERT INTO ${testTable} VALUES (0,3,'2'),(1,25,'25'); """ - sql """ INSERT INTO ${testTable} VALUES (0,3,'3'),(1,26,'26'); """ - sql """ INSERT INTO ${testTable} VALUES (0,4,'4'),(1,27,'27'); """ - sql """ INSERT INTO ${testTable} VALUES (0,5,'5'),(1,28,'28'); """ - - time_diff = System.currentTimeMillis() - now - logger.info("time_diff:" + time_diff) - assertTrue(time_diff <= timeout, "wait_for_insert_into_values timeout") - - qt_sql "select * from ${testTable} order by plan_id" - - GetDebugPoint().disableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed") - } finally { - reset_be_param("compaction_promotion_version_count") - reset_be_param("tablet_rowset_stale_sweep_time_sec") - GetDebugPoint().disableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") - GetDebugPoint().disableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed") - } - -} diff --git a/regression-test/suites/compaction/test_mow_compaction_and_schema_change.groovy b/regression-test/suites/compaction/test_mow_compaction_and_schema_change.groovy index 88b6dcb7d2c0ad..79e637506c78ad 100644 --- a/regression-test/suites/compaction/test_mow_compaction_and_schema_change.groovy +++ b/regression-test/suites/compaction/test_mow_compaction_and_schema_change.groovy @@ -174,24 +174,17 @@ suite("test_mow_compaction_and_schema_change", "nonConcurrent") { GetDebugPoint().clearDebugPointsForAllBEs() get_be_param("tablet_rowset_stale_sweep_time_sec") - get_be_param("enable_delete_bitmap_merge_on_compaction") get_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap") try { set_be_param("tablet_rowset_stale_sweep_time_sec", "0") - for (int method = 0; method < 3; method++) { + for (int method = 0; method < 2; method++) { if (method == 0) { // off - set_be_param("enable_delete_bitmap_merge_on_compaction", "false") set_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap", "false") } else if (method == 1) { // solution2: no duplicated key problems - set_be_param("enable_delete_bitmap_merge_on_compaction", "false") set_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap", "true") - } else if (method == 2) { - // solution1: has duplicated key problems - set_be_param("enable_delete_bitmap_merge_on_compaction", "true") - set_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap", "false") } testTable = "test_mow_compaction_and_schema_change_${method}" @@ -231,7 +224,6 @@ suite("test_mow_compaction_and_schema_change", "nonConcurrent") { order_qt_sql2 "select * from ${testTable}" GetDebugPoint().enableDebugPointForAllBEs("Tablet.delete_expired_stale_rowset.start_delete_unused_rowset") - GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets") // solution 1 GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset") // solution 2 // 3.0 write some data @@ -336,7 +328,6 @@ suite("test_mow_compaction_and_schema_change", "nonConcurrent") { } } finally { reset_be_param("tablet_rowset_stale_sweep_time_sec") - reset_be_param("enable_delete_bitmap_merge_on_compaction") reset_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap") GetDebugPoint().clearDebugPointsForAllBEs() }