From ad1df6fd794aa6645b282c3d9f5983efb4ec0ef3 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 15 Aug 2023 13:11:55 +0800 Subject: [PATCH 1/8] add more info --- be/src/olap/tablet.cpp | 27 ++++++++++++++++++++++----- be/src/olap/tablet.h | 2 +- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 8ed7632bc8f5bb..44f332fe14b118 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -3236,8 +3236,8 @@ Status Tablet::update_delete_bitmap_without_lock(const RowsetSharedPtr& rowset) << "(us), total rows: " << total_rows; if (config::enable_merge_on_write_correctness_check) { // check if all the rowset has ROWSET_SENTINEL_MARK - RETURN_IF_ERROR( - _check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, cur_rowset_ids)); + RETURN_IF_ERROR(_check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, -1, + cur_rowset_ids)); _remove_sentinel_mark_from_delete_bitmap(delete_bitmap); } for (auto iter = delete_bitmap->delete_bitmap.begin(); @@ -3340,8 +3340,8 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, if (config::enable_merge_on_write_correctness_check && rowset->num_rows() != 0) { // only do correctness check if the rowset has at least one row written // check if all the rowset has ROWSET_SENTINEL_MARK - RETURN_IF_ERROR( - _check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, cur_rowset_ids)); + RETURN_IF_ERROR(_check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, txn_id, + cur_rowset_ids)); _remove_sentinel_mark_from_delete_bitmap(delete_bitmap); } @@ -3698,10 +3698,13 @@ void Tablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bit } Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, - const RowsetIdUnorderedSet& rowset_ids) const { + int64_t txn_id, + const RowsetIdUnorderedSet& rowset_ids) { + RowsetIdUnorderedSet missing_ids; for (const auto& rowsetid : rowset_ids) { if (!delete_bitmap->delete_bitmap.contains( {rowsetid, DeleteBitmap::INVALID_SEGMENT_ID, 0})) { + missing_ids.insert(rowsetid); LOG(WARNING) << "check delete bitmap correctness failed, can't find sentinel mark in " "rowset with RowsetId:" << rowsetid << ",max_version:" << max_version; @@ -3709,6 +3712,20 @@ Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, i return Status::OK(); } } + + if (!missing_ids.empty()) { + LOG(WARNING) << "[txn_id:" << txn_id << "][tablet_id:" << tablet_id() + << "][max_version: " << max_version + << "] check delete bitmap correctness failed!"; + std::string rowset_status_string; + get_compaction_status(&rowset_status_string); + LOG(WARNING) << rowset_status_string; + LOG(WARNING) << "can't find sentinel mark in the following rowsets:"; + for (const auto& rowsetid : missing_ids) { + LOG(WARNING) << "[rowsetid: " << rowsetid << "]"; + } + DCHECK(false) << "check delete bitmap correctness failed!"; + } return Status::OK(); } } // namespace doris diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index d78dfe1ff5b4e1..8ca4a0b8dd7038 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -601,7 +601,7 @@ class Tablet : public BaseTablet { void _remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap); Status _check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, - const RowsetIdUnorderedSet& rowset_ids) const; + int64_t txn_id, const RowsetIdUnorderedSet& rowset_ids); public: static const int64_t K_INVALID_CUMULATIVE_POINT = -1; From aab1450a6cbce66c365faec3c5ddd99f8ac3c21d Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 15 Aug 2023 14:00:28 +0800 Subject: [PATCH 2/8] fix --- be/src/olap/tablet.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 44f332fe14b118..5357773eaee88a 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -3705,11 +3705,6 @@ Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, i if (!delete_bitmap->delete_bitmap.contains( {rowsetid, DeleteBitmap::INVALID_SEGMENT_ID, 0})) { missing_ids.insert(rowsetid); - LOG(WARNING) << "check delete bitmap correctness failed, can't find sentinel mark in " - "rowset with RowsetId:" - << rowsetid << ",max_version:" << max_version; - DCHECK(false) << "check delete bitmap correctness failed!"; - return Status::OK(); } } From 95b897e50b6fab617ede195478d6c20ad10ac5c8 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 15 Aug 2023 14:16:24 +0800 Subject: [PATCH 3/8] add lock --- be/src/olap/tablet.cpp | 136 +++++++++++++++++++++++++++++++++++++++-- be/src/olap/tablet.h | 4 +- 2 files changed, 135 insertions(+), 5 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 5357773eaee88a..0927d2c7198459 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1479,6 +1479,129 @@ void Tablet::get_compaction_status(std::string* json_result) { *json_result = std::string(strbuf.GetString()); } +// for printing debug info only +void Tablet::get_compaction_status_without_lock(std::string* json_result) { + rapidjson::Document root; + root.SetObject(); + + rapidjson::Document path_arr; + path_arr.SetArray(); + + std::vector rowsets; + std::vector stale_rowsets; + std::vector delete_flags; + + rowsets.reserve(_rs_version_map.size()); + for (auto& it : _rs_version_map) { + rowsets.push_back(it.second); + } + std::sort(rowsets.begin(), rowsets.end(), Rowset::comparator); + + stale_rowsets.reserve(_stale_rs_version_map.size()); + for (auto& it : _stale_rs_version_map) { + stale_rowsets.push_back(it.second); + } + std::sort(stale_rowsets.begin(), stale_rowsets.end(), Rowset::comparator); + + delete_flags.reserve(rowsets.size()); + for (auto& rs : rowsets) { + delete_flags.push_back(rs->rowset_meta()->has_delete_predicate()); + } + // get snapshot version path json_doc + _timestamped_version_tracker.get_stale_version_path_json_doc(path_arr); + + rapidjson::Value cumulative_policy_type; + std::string policy_type_str = "cumulative compaction policy not initializied"; + if (_cumulative_compaction_policy != nullptr) { + policy_type_str = _cumulative_compaction_policy->name(); + } + cumulative_policy_type.SetString(policy_type_str.c_str(), policy_type_str.length(), + root.GetAllocator()); + root.AddMember("cumulative policy type", cumulative_policy_type, root.GetAllocator()); + root.AddMember("cumulative point", _cumulative_point.load(), root.GetAllocator()); + rapidjson::Value cumu_value; + std::string format_str = ToStringFromUnixMillis(_last_cumu_compaction_failure_millis.load()); + cumu_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); + root.AddMember("last cumulative failure time", cumu_value, root.GetAllocator()); + rapidjson::Value base_value; + format_str = ToStringFromUnixMillis(_last_base_compaction_failure_millis.load()); + base_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); + root.AddMember("last base failure time", base_value, root.GetAllocator()); + rapidjson::Value full_value; + format_str = ToStringFromUnixMillis(_last_full_compaction_failure_millis.load()); + base_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); + root.AddMember("last full failure time", full_value, root.GetAllocator()); + rapidjson::Value cumu_success_value; + format_str = ToStringFromUnixMillis(_last_cumu_compaction_success_millis.load()); + cumu_success_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); + root.AddMember("last cumulative success time", cumu_success_value, root.GetAllocator()); + rapidjson::Value base_success_value; + format_str = ToStringFromUnixMillis(_last_base_compaction_success_millis.load()); + base_success_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); + root.AddMember("last base success time", base_success_value, root.GetAllocator()); + rapidjson::Value full_success_value; + format_str = ToStringFromUnixMillis(_last_full_compaction_success_millis.load()); + full_success_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); + root.AddMember("last full success time", full_success_value, root.GetAllocator()); + + // print all rowsets' version as an array + rapidjson::Document versions_arr; + rapidjson::Document missing_versions_arr; + versions_arr.SetArray(); + missing_versions_arr.SetArray(); + int64_t last_version = -1; + for (int i = 0; i < rowsets.size(); ++i) { + const Version& ver = rowsets[i]->version(); + if (ver.first != last_version + 1) { + rapidjson::Value miss_value; + miss_value.SetString( + strings::Substitute("[$0-$1]", last_version + 1, ver.first - 1).c_str(), + missing_versions_arr.GetAllocator()); + missing_versions_arr.PushBack(miss_value, missing_versions_arr.GetAllocator()); + } + rapidjson::Value value; + std::string disk_size = PrettyPrinter::print( + static_cast(rowsets[i]->rowset_meta()->total_disk_size()), TUnit::BYTES); + std::string version_str = strings::Substitute( + "[$0-$1] $2 $3 $4 $5 $6", ver.first, ver.second, rowsets[i]->num_segments(), + (delete_flags[i] ? "DELETE" : "DATA"), + SegmentsOverlapPB_Name(rowsets[i]->rowset_meta()->segments_overlap()), + rowsets[i]->rowset_id().to_string(), disk_size); + value.SetString(version_str.c_str(), version_str.length(), versions_arr.GetAllocator()); + versions_arr.PushBack(value, versions_arr.GetAllocator()); + last_version = ver.second; + } + root.AddMember("rowsets", versions_arr, root.GetAllocator()); + root.AddMember("missing_rowsets", missing_versions_arr, root.GetAllocator()); + + // print all stale rowsets' version as an array + rapidjson::Document stale_versions_arr; + stale_versions_arr.SetArray(); + for (int i = 0; i < stale_rowsets.size(); ++i) { + const Version& ver = stale_rowsets[i]->version(); + rapidjson::Value value; + std::string disk_size = PrettyPrinter::print( + static_cast(stale_rowsets[i]->rowset_meta()->total_disk_size()), + TUnit::BYTES); + std::string version_str = strings::Substitute( + "[$0-$1] $2 $3 $4", ver.first, ver.second, stale_rowsets[i]->num_segments(), + stale_rowsets[i]->rowset_id().to_string(), disk_size); + value.SetString(version_str.c_str(), version_str.length(), + stale_versions_arr.GetAllocator()); + stale_versions_arr.PushBack(value, stale_versions_arr.GetAllocator()); + } + root.AddMember("stale_rowsets", stale_versions_arr, root.GetAllocator()); + + // add stale version rowsets + root.AddMember("stale version path", path_arr, root.GetAllocator()); + + // to json string + rapidjson::StringBuffer strbuf; + rapidjson::PrettyWriter writer(strbuf); + root.Accept(writer); + *json_result = std::string(strbuf.GetString()); +} + bool Tablet::do_tablet_meta_checkpoint() { std::lock_guard store_lock(_meta_store_lock); if (_newly_created_rowset_num == 0) { @@ -3237,7 +3360,7 @@ Status Tablet::update_delete_bitmap_without_lock(const RowsetSharedPtr& rowset) if (config::enable_merge_on_write_correctness_check) { // check if all the rowset has ROWSET_SENTINEL_MARK RETURN_IF_ERROR(_check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, -1, - cur_rowset_ids)); + cur_rowset_ids, true)); _remove_sentinel_mark_from_delete_bitmap(delete_bitmap); } for (auto iter = delete_bitmap->delete_bitmap.begin(); @@ -3341,7 +3464,7 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, // only do correctness check if the rowset has at least one row written // check if all the rowset has ROWSET_SENTINEL_MARK RETURN_IF_ERROR(_check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, txn_id, - cur_rowset_ids)); + cur_rowset_ids, false)); _remove_sentinel_mark_from_delete_bitmap(delete_bitmap); } @@ -3699,7 +3822,8 @@ void Tablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bit Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, int64_t txn_id, - const RowsetIdUnorderedSet& rowset_ids) { + const RowsetIdUnorderedSet& rowset_ids, + bool with_meta_lock) { RowsetIdUnorderedSet missing_ids; for (const auto& rowsetid : rowset_ids) { if (!delete_bitmap->delete_bitmap.contains( @@ -3713,7 +3837,11 @@ Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, i << "][max_version: " << max_version << "] check delete bitmap correctness failed!"; std::string rowset_status_string; - get_compaction_status(&rowset_status_string); + if (with_meta_lock) { + get_compaction_status_without_lock(&rowset_status_string); + } else { + get_compaction_status(&rowset_status_string); + } LOG(WARNING) << rowset_status_string; LOG(WARNING) << "can't find sentinel mark in the following rowsets:"; for (const auto& rowsetid : missing_ids) { diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 8ca4a0b8dd7038..dcd855d1b5e467 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -303,6 +303,7 @@ class Tablet : public BaseTablet { // return a json string to show the compaction status of this tablet void get_compaction_status(std::string* json_result); + void get_compaction_status_without_lock(std::string* json_result); Status prepare_compaction_and_calculate_permits(CompactionType compaction_type, TabletSharedPtr tablet, int64_t* permits); @@ -601,7 +602,8 @@ class Tablet : public BaseTablet { void _remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap); Status _check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, - int64_t txn_id, const RowsetIdUnorderedSet& rowset_ids); + int64_t txn_id, const RowsetIdUnorderedSet& rowset_ids, + bool with_meta_lock); public: static const int64_t K_INVALID_CUMULATIVE_POINT = -1; From 5a9371fc07ee90a4c64489fa54b3bb8b5031d7e6 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 15 Aug 2023 16:02:39 +0800 Subject: [PATCH 4/8] update --- be/src/olap/tablet.cpp | 244 +++++++++++++++-------------------------- be/src/olap/tablet.h | 9 ++ 2 files changed, 96 insertions(+), 157 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 0927d2c7198459..d3e7b0b3d951e0 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1355,38 +1355,37 @@ std::vector Tablet::pick_candidate_rowsets_to_build_inverted_in return candidate_rowsets; } -// For http compaction action -void Tablet::get_compaction_status(std::string* json_result) { - rapidjson::Document root; - root.SetObject(); - - rapidjson::Document path_arr; - path_arr.SetArray(); +void Tablet::extract_rowsets_unlocked(std::vector& rowsets) { + rowsets.reserve(_rs_version_map.size()); + for (auto& it : _rs_version_map) { + rowsets.push_back(it.second); + } + std::sort(rowsets.begin(), rowsets.end(), Rowset::comparator); +} - std::vector rowsets; - std::vector stale_rowsets; - std::vector delete_flags; - { - std::shared_lock rdlock(_meta_lock); - rowsets.reserve(_rs_version_map.size()); - for (auto& it : _rs_version_map) { - rowsets.push_back(it.second); - } - std::sort(rowsets.begin(), rowsets.end(), Rowset::comparator); +void Tablet::extract_stale_rowsets_unlocked(std::vector& stale_rowsets) { + stale_rowsets.reserve(_stale_rs_version_map.size()); + for (auto& it : _stale_rs_version_map) { + stale_rowsets.push_back(it.second); + } + std::sort(stale_rowsets.begin(), stale_rowsets.end(), Rowset::comparator); +} - stale_rowsets.reserve(_stale_rs_version_map.size()); - for (auto& it : _stale_rs_version_map) { - stale_rowsets.push_back(it.second); - } - std::sort(stale_rowsets.begin(), stale_rowsets.end(), Rowset::comparator); +std::string Tablet::get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag) { + const Version& ver = rowset->version(); + std::string disk_size = PrettyPrinter::print( + static_cast(rowset->rowset_meta()->total_disk_size()), TUnit::BYTES); + return strings::Substitute("[$0-$1] $2 $3 $4 $5 $6", ver.first, ver.second, + rowset->num_segments(), (delete_flag ? "DELETE" : "DATA"), + SegmentsOverlapPB_Name(rowset->rowset_meta()->segments_overlap()), + rowset->rowset_id().to_string(), disk_size); +} - delete_flags.reserve(rowsets.size()); - for (auto& rs : rowsets) { - delete_flags.push_back(rs->rowset_meta()->has_delete_predicate()); - } - // get snapshot version path json_doc - _timestamped_version_tracker.get_stale_version_path_json_doc(path_arr); - } +void Tablet::get_rowsets_info_pretty_json(rapidjson::Document& root, rapidjson::Document& path_arr, + const std::vector& rowsets, + const std::vector& stale_rowsets, + const std::vector& delete_flags, + std::string* json_result) { rapidjson::Value cumulative_policy_type; std::string policy_type_str = "cumulative compaction policy not initializied"; if (_cumulative_compaction_policy != nullptr) { @@ -1437,14 +1436,7 @@ void Tablet::get_compaction_status(std::string* json_result) { missing_versions_arr.PushBack(miss_value, missing_versions_arr.GetAllocator()); } rapidjson::Value value; - std::string disk_size = PrettyPrinter::print( - static_cast(rowsets[i]->rowset_meta()->total_disk_size()), TUnit::BYTES); - std::string version_str = strings::Substitute( - "[$0-$1] $2 $3 $4 $5 $6", ver.first, ver.second, rowsets[i]->num_segments(), - (delete_flags[i] ? "DELETE" : "DATA"), - SegmentsOverlapPB_Name(rowsets[i]->rowset_meta()->segments_overlap()), - rowsets[i]->rowset_id().to_string(), disk_size); - value.SetString(version_str.c_str(), version_str.length(), versions_arr.GetAllocator()); + std::string version_str = get_rowset_info_str(rowsets[i], delete_flags[i]); versions_arr.PushBack(value, versions_arr.GetAllocator()); last_version = ver.second; } @@ -1479,8 +1471,8 @@ void Tablet::get_compaction_status(std::string* json_result) { *json_result = std::string(strbuf.GetString()); } -// for printing debug info only -void Tablet::get_compaction_status_without_lock(std::string* json_result) { +// For http compaction action +void Tablet::get_compaction_status(std::string* json_result) { rapidjson::Document root; root.SetObject(); @@ -1490,116 +1482,19 @@ void Tablet::get_compaction_status_without_lock(std::string* json_result) { std::vector rowsets; std::vector stale_rowsets; std::vector delete_flags; + { + std::shared_lock rdlock(_meta_lock); + extract_rowsets_unlocked(rowsets); + extract_stale_rowsets_unlocked(stale_rowsets); - rowsets.reserve(_rs_version_map.size()); - for (auto& it : _rs_version_map) { - rowsets.push_back(it.second); - } - std::sort(rowsets.begin(), rowsets.end(), Rowset::comparator); - - stale_rowsets.reserve(_stale_rs_version_map.size()); - for (auto& it : _stale_rs_version_map) { - stale_rowsets.push_back(it.second); - } - std::sort(stale_rowsets.begin(), stale_rowsets.end(), Rowset::comparator); - - delete_flags.reserve(rowsets.size()); - for (auto& rs : rowsets) { - delete_flags.push_back(rs->rowset_meta()->has_delete_predicate()); - } - // get snapshot version path json_doc - _timestamped_version_tracker.get_stale_version_path_json_doc(path_arr); - - rapidjson::Value cumulative_policy_type; - std::string policy_type_str = "cumulative compaction policy not initializied"; - if (_cumulative_compaction_policy != nullptr) { - policy_type_str = _cumulative_compaction_policy->name(); - } - cumulative_policy_type.SetString(policy_type_str.c_str(), policy_type_str.length(), - root.GetAllocator()); - root.AddMember("cumulative policy type", cumulative_policy_type, root.GetAllocator()); - root.AddMember("cumulative point", _cumulative_point.load(), root.GetAllocator()); - rapidjson::Value cumu_value; - std::string format_str = ToStringFromUnixMillis(_last_cumu_compaction_failure_millis.load()); - cumu_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); - root.AddMember("last cumulative failure time", cumu_value, root.GetAllocator()); - rapidjson::Value base_value; - format_str = ToStringFromUnixMillis(_last_base_compaction_failure_millis.load()); - base_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); - root.AddMember("last base failure time", base_value, root.GetAllocator()); - rapidjson::Value full_value; - format_str = ToStringFromUnixMillis(_last_full_compaction_failure_millis.load()); - base_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); - root.AddMember("last full failure time", full_value, root.GetAllocator()); - rapidjson::Value cumu_success_value; - format_str = ToStringFromUnixMillis(_last_cumu_compaction_success_millis.load()); - cumu_success_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); - root.AddMember("last cumulative success time", cumu_success_value, root.GetAllocator()); - rapidjson::Value base_success_value; - format_str = ToStringFromUnixMillis(_last_base_compaction_success_millis.load()); - base_success_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); - root.AddMember("last base success time", base_success_value, root.GetAllocator()); - rapidjson::Value full_success_value; - format_str = ToStringFromUnixMillis(_last_full_compaction_success_millis.load()); - full_success_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator()); - root.AddMember("last full success time", full_success_value, root.GetAllocator()); - - // print all rowsets' version as an array - rapidjson::Document versions_arr; - rapidjson::Document missing_versions_arr; - versions_arr.SetArray(); - missing_versions_arr.SetArray(); - int64_t last_version = -1; - for (int i = 0; i < rowsets.size(); ++i) { - const Version& ver = rowsets[i]->version(); - if (ver.first != last_version + 1) { - rapidjson::Value miss_value; - miss_value.SetString( - strings::Substitute("[$0-$1]", last_version + 1, ver.first - 1).c_str(), - missing_versions_arr.GetAllocator()); - missing_versions_arr.PushBack(miss_value, missing_versions_arr.GetAllocator()); + delete_flags.reserve(rowsets.size()); + for (auto& rs : rowsets) { + delete_flags.push_back(rs->rowset_meta()->has_delete_predicate()); } - rapidjson::Value value; - std::string disk_size = PrettyPrinter::print( - static_cast(rowsets[i]->rowset_meta()->total_disk_size()), TUnit::BYTES); - std::string version_str = strings::Substitute( - "[$0-$1] $2 $3 $4 $5 $6", ver.first, ver.second, rowsets[i]->num_segments(), - (delete_flags[i] ? "DELETE" : "DATA"), - SegmentsOverlapPB_Name(rowsets[i]->rowset_meta()->segments_overlap()), - rowsets[i]->rowset_id().to_string(), disk_size); - value.SetString(version_str.c_str(), version_str.length(), versions_arr.GetAllocator()); - versions_arr.PushBack(value, versions_arr.GetAllocator()); - last_version = ver.second; - } - root.AddMember("rowsets", versions_arr, root.GetAllocator()); - root.AddMember("missing_rowsets", missing_versions_arr, root.GetAllocator()); - - // print all stale rowsets' version as an array - rapidjson::Document stale_versions_arr; - stale_versions_arr.SetArray(); - for (int i = 0; i < stale_rowsets.size(); ++i) { - const Version& ver = stale_rowsets[i]->version(); - rapidjson::Value value; - std::string disk_size = PrettyPrinter::print( - static_cast(stale_rowsets[i]->rowset_meta()->total_disk_size()), - TUnit::BYTES); - std::string version_str = strings::Substitute( - "[$0-$1] $2 $3 $4", ver.first, ver.second, stale_rowsets[i]->num_segments(), - stale_rowsets[i]->rowset_id().to_string(), disk_size); - value.SetString(version_str.c_str(), version_str.length(), - stale_versions_arr.GetAllocator()); - stale_versions_arr.PushBack(value, stale_versions_arr.GetAllocator()); + // get snapshot version path json_doc + _timestamped_version_tracker.get_stale_version_path_json_doc(path_arr); } - root.AddMember("stale_rowsets", stale_versions_arr, root.GetAllocator()); - - // add stale version rowsets - root.AddMember("stale version path", path_arr, root.GetAllocator()); - - // to json string - rapidjson::StringBuffer strbuf; - rapidjson::PrettyWriter writer(strbuf); - root.Accept(writer); - *json_result = std::string(strbuf.GetString()); + get_rowsets_info_pretty_json(root, path_arr, rowsets, stale_rowsets, delete_flags, json_result); } bool Tablet::do_tablet_meta_checkpoint() { @@ -3360,7 +3255,7 @@ Status Tablet::update_delete_bitmap_without_lock(const RowsetSharedPtr& rowset) if (config::enable_merge_on_write_correctness_check) { // check if all the rowset has ROWSET_SENTINEL_MARK RETURN_IF_ERROR(_check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, -1, - cur_rowset_ids, true)); + cur_rowset_ids, specified_rowsets, true)); _remove_sentinel_mark_from_delete_bitmap(delete_bitmap); } for (auto iter = delete_bitmap->delete_bitmap.begin(); @@ -3464,7 +3359,7 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, // only do correctness check if the rowset has at least one row written // check if all the rowset has ROWSET_SENTINEL_MARK RETURN_IF_ERROR(_check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, txn_id, - cur_rowset_ids, false)); + cur_rowset_ids, specified_rowsets, false)); _remove_sentinel_mark_from_delete_bitmap(delete_bitmap); } @@ -3823,6 +3718,7 @@ void Tablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bit Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, int64_t txn_id, const RowsetIdUnorderedSet& rowset_ids, + const std::vector& rowsets, bool with_meta_lock) { RowsetIdUnorderedSet missing_ids; for (const auto& rowsetid : rowset_ids) { @@ -3836,17 +3732,51 @@ Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, i LOG(WARNING) << "[txn_id:" << txn_id << "][tablet_id:" << tablet_id() << "][max_version: " << max_version << "] check delete bitmap correctness failed!"; - std::string rowset_status_string; - if (with_meta_lock) { - get_compaction_status_without_lock(&rowset_status_string); - } else { - get_compaction_status(&rowset_status_string); + std::vector delete_flags; + delete_flags.reserve(rowsets.size()); + for (auto& rs : rowsets) { + delete_flags.push_back(rs->rowset_meta()->has_delete_predicate()); } - LOG(WARNING) << rowset_status_string; - LOG(WARNING) << "can't find sentinel mark in the following rowsets:"; - for (const auto& rowsetid : missing_ids) { - LOG(WARNING) << "[rowsetid: " << rowsetid << "]"; + rapidjson::Document root; + root.SetObject(); + rapidjson::Document required_rowsets_arr; + required_rowsets_arr.SetArray(); + rapidjson::Document missing_rowsets_arr; + missing_rowsets_arr.SetArray(); + rapidjson::Document missing_versions_arr; + missing_versions_arr.SetArray(); + + int64_t last_version = -1; + for (int i = 0; i < rowsets.size(); ++i) { + const Version& ver = rowsets[i]->version(); + if (ver.first != last_version + 1) { + rapidjson::Value miss_value; + miss_value.SetString( + strings::Substitute("[$0-$1]", last_version + 1, ver.first - 1).c_str(), + missing_versions_arr.GetAllocator()); + missing_versions_arr.PushBack(miss_value, missing_versions_arr.GetAllocator()); + } + rapidjson::Value value; + std::string version_str = get_rowset_info_str(rowsets[i], delete_flags[i]); + required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator()); + last_version = ver.second; } + for (const auto& missing_rowset_id : missing_ids) { + rapidjson::Value miss_value; + std::string rowset_id_str = missing_rowset_id.to_string(); + miss_value.SetString(rowset_id_str.c_str(), rowset_id_str.length(), + missing_rowsets_arr.GetAllocator()); + missing_rowsets_arr.PushBack(miss_value, missing_rowsets_arr.GetAllocator()); + } + + root.AddMember("requied_rowsets", required_rowsets_arr, root.GetAllocator()); + root.AddMember("missing_rowsets", missing_rowsets_arr, root.GetAllocator()); + root.AddMember("missing_versions", missing_versions_arr, root.GetAllocator()); + rapidjson::StringBuffer strbuf; + rapidjson::PrettyWriter writer(strbuf); + root.Accept(writer); + std::string rowset_status_string = std::string(strbuf.GetString()); + LOG_EVERY_SECOND(WARNING) << rowset_status_string; DCHECK(false) << "check delete bitmap correctness failed!"; } return Status::OK(); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index dcd855d1b5e467..d50ade3533ba9d 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -301,6 +301,14 @@ class Tablet : public BaseTablet { // caller should hold the _meta_lock before calling this method void generate_tablet_meta_copy_unlocked(TabletMetaSharedPtr new_tablet_meta) const; + void extract_rowsets_unlocked(std::vector& rowsets); + void extract_stale_rowsets_unlocked(std::vector& stale_rowsets); + std::string get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag); + void get_rowsets_info_pretty_json(rapidjson::Document& root, rapidjson::Document& path_arr, + const std::vector& rowsets, + const std::vector& stale_rowsets, + const std::vector& delete_flags, + std::string* json_result); // return a json string to show the compaction status of this tablet void get_compaction_status(std::string* json_result); void get_compaction_status_without_lock(std::string* json_result); @@ -603,6 +611,7 @@ class Tablet : public BaseTablet { void _remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap); Status _check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, int64_t txn_id, const RowsetIdUnorderedSet& rowset_ids, + const std::vector& rowsets, bool with_meta_lock); public: From 1cb5a91b783c4c1c7b0c2affd54e00b6bc4a5645 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 15 Aug 2023 16:38:50 +0800 Subject: [PATCH 5/8] update --- be/src/olap/tablet.cpp | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index d3e7b0b3d951e0..645526d363432a 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1437,6 +1437,7 @@ void Tablet::get_rowsets_info_pretty_json(rapidjson::Document& root, rapidjson:: } rapidjson::Value value; std::string version_str = get_rowset_info_str(rowsets[i], delete_flags[i]); + value.SetString(version_str.c_str(), version_str.length(), versions_arr.GetAllocator()); versions_arr.PushBack(value, versions_arr.GetAllocator()); last_version = ver.second; } @@ -3743,23 +3744,13 @@ Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, i required_rowsets_arr.SetArray(); rapidjson::Document missing_rowsets_arr; missing_rowsets_arr.SetArray(); - rapidjson::Document missing_versions_arr; - missing_versions_arr.SetArray(); - int64_t last_version = -1; for (int i = 0; i < rowsets.size(); ++i) { - const Version& ver = rowsets[i]->version(); - if (ver.first != last_version + 1) { - rapidjson::Value miss_value; - miss_value.SetString( - strings::Substitute("[$0-$1]", last_version + 1, ver.first - 1).c_str(), - missing_versions_arr.GetAllocator()); - missing_versions_arr.PushBack(miss_value, missing_versions_arr.GetAllocator()); - } rapidjson::Value value; std::string version_str = get_rowset_info_str(rowsets[i], delete_flags[i]); + value.SetString(version_str.c_str(), version_str.length(), + required_rowsets_arr.GetAllocator()); required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator()); - last_version = ver.second; } for (const auto& missing_rowset_id : missing_ids) { rapidjson::Value miss_value; @@ -3771,7 +3762,6 @@ Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, i root.AddMember("requied_rowsets", required_rowsets_arr, root.GetAllocator()); root.AddMember("missing_rowsets", missing_rowsets_arr, root.GetAllocator()); - root.AddMember("missing_versions", missing_versions_arr, root.GetAllocator()); rapidjson::StringBuffer strbuf; rapidjson::PrettyWriter writer(strbuf); root.Accept(writer); From eefdf95593614820954356a6a4fee71c075d859d Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 15 Aug 2023 18:18:17 +0800 Subject: [PATCH 6/8] update --- be/src/olap/tablet.cpp | 79 +++++++++++++++++------------------------- be/src/olap/tablet.h | 8 ----- 2 files changed, 32 insertions(+), 55 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 645526d363432a..f8bca3ba6d2ac4 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1355,22 +1355,6 @@ std::vector Tablet::pick_candidate_rowsets_to_build_inverted_in return candidate_rowsets; } -void Tablet::extract_rowsets_unlocked(std::vector& rowsets) { - rowsets.reserve(_rs_version_map.size()); - for (auto& it : _rs_version_map) { - rowsets.push_back(it.second); - } - std::sort(rowsets.begin(), rowsets.end(), Rowset::comparator); -} - -void Tablet::extract_stale_rowsets_unlocked(std::vector& stale_rowsets) { - stale_rowsets.reserve(_stale_rs_version_map.size()); - for (auto& it : _stale_rs_version_map) { - stale_rowsets.push_back(it.second); - } - std::sort(stale_rowsets.begin(), stale_rowsets.end(), Rowset::comparator); -} - std::string Tablet::get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag) { const Version& ver = rowset->version(); std::string disk_size = PrettyPrinter::print( @@ -1381,11 +1365,38 @@ std::string Tablet::get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag rowset->rowset_id().to_string(), disk_size); } -void Tablet::get_rowsets_info_pretty_json(rapidjson::Document& root, rapidjson::Document& path_arr, - const std::vector& rowsets, - const std::vector& stale_rowsets, - const std::vector& delete_flags, - std::string* json_result) { +// For http compaction action +void Tablet::get_compaction_status(std::string* json_result) { + rapidjson::Document root; + root.SetObject(); + + rapidjson::Document path_arr; + path_arr.SetArray(); + + std::vector rowsets; + std::vector stale_rowsets; + std::vector delete_flags; + { + std::shared_lock rdlock(_meta_lock); + rowsets.reserve(_rs_version_map.size()); + for (auto& it : _rs_version_map) { + rowsets.push_back(it.second); + } + std::sort(rowsets.begin(), rowsets.end(), Rowset::comparator); + + stale_rowsets.reserve(_stale_rs_version_map.size()); + for (auto& it : _stale_rs_version_map) { + stale_rowsets.push_back(it.second); + } + std::sort(stale_rowsets.begin(), stale_rowsets.end(), Rowset::comparator); + + delete_flags.reserve(rowsets.size()); + for (auto& rs : rowsets) { + delete_flags.push_back(rs->rowset_meta()->has_delete_predicate()); + } + // get snapshot version path json_doc + _timestamped_version_tracker.get_stale_version_path_json_doc(path_arr); + } rapidjson::Value cumulative_policy_type; std::string policy_type_str = "cumulative compaction policy not initializied"; if (_cumulative_compaction_policy != nullptr) { @@ -1472,32 +1483,6 @@ void Tablet::get_rowsets_info_pretty_json(rapidjson::Document& root, rapidjson:: *json_result = std::string(strbuf.GetString()); } -// For http compaction action -void Tablet::get_compaction_status(std::string* json_result) { - rapidjson::Document root; - root.SetObject(); - - rapidjson::Document path_arr; - path_arr.SetArray(); - - std::vector rowsets; - std::vector stale_rowsets; - std::vector delete_flags; - { - std::shared_lock rdlock(_meta_lock); - extract_rowsets_unlocked(rowsets); - extract_stale_rowsets_unlocked(stale_rowsets); - - delete_flags.reserve(rowsets.size()); - for (auto& rs : rowsets) { - delete_flags.push_back(rs->rowset_meta()->has_delete_predicate()); - } - // get snapshot version path json_doc - _timestamped_version_tracker.get_stale_version_path_json_doc(path_arr); - } - get_rowsets_info_pretty_json(root, path_arr, rowsets, stale_rowsets, delete_flags, json_result); -} - bool Tablet::do_tablet_meta_checkpoint() { std::lock_guard store_lock(_meta_store_lock); if (_newly_created_rowset_num == 0) { diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index d50ade3533ba9d..d876c150227fbe 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -301,17 +301,9 @@ class Tablet : public BaseTablet { // caller should hold the _meta_lock before calling this method void generate_tablet_meta_copy_unlocked(TabletMetaSharedPtr new_tablet_meta) const; - void extract_rowsets_unlocked(std::vector& rowsets); - void extract_stale_rowsets_unlocked(std::vector& stale_rowsets); std::string get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag); - void get_rowsets_info_pretty_json(rapidjson::Document& root, rapidjson::Document& path_arr, - const std::vector& rowsets, - const std::vector& stale_rowsets, - const std::vector& delete_flags, - std::string* json_result); // return a json string to show the compaction status of this tablet void get_compaction_status(std::string* json_result); - void get_compaction_status_without_lock(std::string* json_result); Status prepare_compaction_and_calculate_permits(CompactionType compaction_type, TabletSharedPtr tablet, int64_t* permits); From 41c270b12bd6f0d41087122fc4514a72a5d5abca Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 15 Aug 2023 22:22:38 +0800 Subject: [PATCH 7/8] update --- be/src/olap/tablet.cpp | 45 ++++++++++++++++++++++++++---------------- be/src/olap/tablet.h | 5 ++--- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index f8bca3ba6d2ac4..748d59a9334416 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1355,7 +1355,7 @@ std::vector Tablet::pick_candidate_rowsets_to_build_inverted_in return candidate_rowsets; } -std::string Tablet::get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag) { +std::string Tablet::_get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag) { const Version& ver = rowset->version(); std::string disk_size = PrettyPrinter::print( static_cast(rowset->rowset_meta()->total_disk_size()), TUnit::BYTES); @@ -1447,7 +1447,7 @@ void Tablet::get_compaction_status(std::string* json_result) { missing_versions_arr.PushBack(miss_value, missing_versions_arr.GetAllocator()); } rapidjson::Value value; - std::string version_str = get_rowset_info_str(rowsets[i], delete_flags[i]); + std::string version_str = _get_rowset_info_str(rowsets[i], delete_flags[i]); value.SetString(version_str.c_str(), version_str.length(), versions_arr.GetAllocator()); versions_arr.PushBack(value, versions_arr.GetAllocator()); last_version = ver.second; @@ -3241,7 +3241,7 @@ Status Tablet::update_delete_bitmap_without_lock(const RowsetSharedPtr& rowset) if (config::enable_merge_on_write_correctness_check) { // check if all the rowset has ROWSET_SENTINEL_MARK RETURN_IF_ERROR(_check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, -1, - cur_rowset_ids, specified_rowsets, true)); + cur_rowset_ids, specified_rowsets)); _remove_sentinel_mark_from_delete_bitmap(delete_bitmap); } for (auto iter = delete_bitmap->delete_bitmap.begin(); @@ -3345,7 +3345,7 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, // only do correctness check if the rowset has at least one row written // check if all the rowset has ROWSET_SENTINEL_MARK RETURN_IF_ERROR(_check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, txn_id, - cur_rowset_ids, specified_rowsets, false)); + cur_rowset_ids)); _remove_sentinel_mark_from_delete_bitmap(delete_bitmap); } @@ -3704,8 +3704,7 @@ void Tablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bit Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, int64_t txn_id, const RowsetIdUnorderedSet& rowset_ids, - const std::vector& rowsets, - bool with_meta_lock) { + const std::vector* rowsets) { RowsetIdUnorderedSet missing_ids; for (const auto& rowsetid : rowset_ids) { if (!delete_bitmap->delete_bitmap.contains( @@ -3718,11 +3717,6 @@ Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, i LOG(WARNING) << "[txn_id:" << txn_id << "][tablet_id:" << tablet_id() << "][max_version: " << max_version << "] check delete bitmap correctness failed!"; - std::vector delete_flags; - delete_flags.reserve(rowsets.size()); - for (auto& rs : rowsets) { - delete_flags.push_back(rs->rowset_meta()->has_delete_predicate()); - } rapidjson::Document root; root.SetObject(); rapidjson::Document required_rowsets_arr; @@ -3730,12 +3724,29 @@ Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, i rapidjson::Document missing_rowsets_arr; missing_rowsets_arr.SetArray(); - for (int i = 0; i < rowsets.size(); ++i) { - rapidjson::Value value; - std::string version_str = get_rowset_info_str(rowsets[i], delete_flags[i]); - value.SetString(version_str.c_str(), version_str.length(), - required_rowsets_arr.GetAllocator()); - required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator()); + if (rowsets != nullptr) { + for (const auto& rowset : *rowsets) { + rapidjson::Value value; + std::string version_str = + _get_rowset_info_str(rowset, rowset.rowset_meta()->has_delete_predicate()); + value.SetString(version_str.c_str(), version_str.length(), + required_rowsets_arr.GetAllocator()); + required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator()); + } + } else { + std::vector rowsets; + { + std::shared_lock meta_rlock(_meta_lock); + rowsets = get_rowset_by_ids(&rowset_ids_to_add); + } + for (const auto& rowset : rowsets) { + rapidjson::Value value; + std::string version_str = + _get_rowset_info_str(rowset, rowset.rowset_meta()->has_delete_predicate()); + value.SetString(version_str.c_str(), version_str.length(), + required_rowsets_arr.GetAllocator()); + required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator()); + } } for (const auto& missing_rowset_id : missing_ids) { rapidjson::Value miss_value; diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index d876c150227fbe..8b781c8f48d8bf 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -301,7 +301,6 @@ class Tablet : public BaseTablet { // caller should hold the _meta_lock before calling this method void generate_tablet_meta_copy_unlocked(TabletMetaSharedPtr new_tablet_meta) const; - std::string get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag); // return a json string to show the compaction status of this tablet void get_compaction_status(std::string* json_result); @@ -603,8 +602,8 @@ class Tablet : public BaseTablet { void _remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap); Status _check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, int64_t txn_id, const RowsetIdUnorderedSet& rowset_ids, - const std::vector& rowsets, - bool with_meta_lock); + const std::vector* rowsets = nullptr); + std::string _get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag); public: static const int64_t K_INVALID_CUMULATIVE_POINT = -1; From aa504626ee5cdfdd96a705cac786ce6ea55ff834 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 16 Aug 2023 15:02:15 +0800 Subject: [PATCH 8/8] fix --- be/src/olap/tablet.cpp | 10 +++++----- be/src/olap/tablet.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 748d59a9334416..66890a355140fd 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -3241,7 +3241,7 @@ Status Tablet::update_delete_bitmap_without_lock(const RowsetSharedPtr& rowset) if (config::enable_merge_on_write_correctness_check) { // check if all the rowset has ROWSET_SENTINEL_MARK RETURN_IF_ERROR(_check_delete_bitmap_correctness(delete_bitmap, cur_version - 1, -1, - cur_rowset_ids, specified_rowsets)); + cur_rowset_ids, &specified_rowsets)); _remove_sentinel_mark_from_delete_bitmap(delete_bitmap); } for (auto iter = delete_bitmap->delete_bitmap.begin(); @@ -3704,7 +3704,7 @@ void Tablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bit Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, int64_t txn_id, const RowsetIdUnorderedSet& rowset_ids, - const std::vector* rowsets) { + std::vector* rowsets) { RowsetIdUnorderedSet missing_ids; for (const auto& rowsetid : rowset_ids) { if (!delete_bitmap->delete_bitmap.contains( @@ -3728,7 +3728,7 @@ Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, i for (const auto& rowset : *rowsets) { rapidjson::Value value; std::string version_str = - _get_rowset_info_str(rowset, rowset.rowset_meta()->has_delete_predicate()); + _get_rowset_info_str(rowset, rowset->rowset_meta()->has_delete_predicate()); value.SetString(version_str.c_str(), version_str.length(), required_rowsets_arr.GetAllocator()); required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator()); @@ -3737,12 +3737,12 @@ Status Tablet::_check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, i std::vector rowsets; { std::shared_lock meta_rlock(_meta_lock); - rowsets = get_rowset_by_ids(&rowset_ids_to_add); + rowsets = get_rowset_by_ids(&rowset_ids); } for (const auto& rowset : rowsets) { rapidjson::Value value; std::string version_str = - _get_rowset_info_str(rowset, rowset.rowset_meta()->has_delete_predicate()); + _get_rowset_info_str(rowset, rowset->rowset_meta()->has_delete_predicate()); value.SetString(version_str.c_str(), version_str.length(), required_rowsets_arr.GetAllocator()); required_rowsets_arr.PushBack(value, required_rowsets_arr.GetAllocator()); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 8b781c8f48d8bf..d698e9ba3e7b08 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -602,7 +602,7 @@ class Tablet : public BaseTablet { void _remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete_bitmap); Status _check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, int64_t txn_id, const RowsetIdUnorderedSet& rowset_ids, - const std::vector* rowsets = nullptr); + std::vector* rowsets = nullptr); std::string _get_rowset_info_str(RowsetSharedPtr rowset, bool delete_flag); public: