From 01d0c2475199dd8522b5dfbd1fcfb25f10e8fb3e Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Tue, 4 Jun 2024 15:55:27 +0800 Subject: [PATCH 1/4] [improve](cloud) use delete bitmap cache to reduce sync rowset time --- be/src/cloud/cloud_meta_mgr.cpp | 37 +++++++++++++++++++ be/src/cloud/cloud_tablet.cpp | 16 ++++++-- .../cloud/cloud_txn_delete_bitmap_cache.cpp | 21 +++++++++++ be/src/cloud/cloud_txn_delete_bitmap_cache.h | 2 + be/src/olap/tablet_meta.h | 2 + 5 files changed, 75 insertions(+), 3 deletions(-) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index d55c884a6c2b7d..d1f8f3033efabd 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -35,6 +35,7 @@ #include #include +#include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/config.h" #include "cloud/pb_convert.h" @@ -50,6 +51,7 @@ #include "olap/olap_common.h" #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_factory.h" +#include "olap/storage_engine.h" #include "olap/tablet_meta.h" #include "runtime/client_cache.h" #include "runtime/exec_env.h" @@ -554,6 +556,41 @@ Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_ return Status::OK(); } + { + auto txn_id = rs_metas.rbegin()->txn_id(); + RowsetSharedPtr rowset; + DeleteBitmapPtr tmp_delete_bitmap; + RowsetIdUnorderedSet rowset_ids; + std::shared_ptr partial_update_info; + int64_t txn_expiration; + CloudStorageEngine& engine = ExecEnv::GetInstance()->storage_engine().to_cloud(); + Status status = engine.txn_delete_bitmap_cache().get_tablet_txn_info( + txn_id, tablet->tablet_id(), &rowset, &tmp_delete_bitmap, &rowset_ids, + &txn_expiration, &partial_update_info); + if (status.ok()) { + DeleteBitmapPtr new_delete_bitmap = std::make_shared(tablet->tablet_id()); + for (auto iter = tmp_delete_bitmap->delete_bitmap.begin(); + iter != tmp_delete_bitmap->delete_bitmap.end(); ++iter) { + DeleteBitmap::BitmapKey key = {std::get<0>(iter->first), std::get<1>(iter->first), + old_max_version + 1}; + LOG(INFO) << "key " << std::get<0>(iter->first) << "," << std::get<1>(iter->first) + << "," << old_max_version + 1; + auto it = tmp_delete_bitmap->status_map.find(key); + if (it != tmp_delete_bitmap->status_map.end() && + it->second == DeleteBitmap::PublishStatus::FINALIZED) { + new_delete_bitmap->merge(key, iter->second); + } + } + *delete_bitmap = *new_delete_bitmap; + engine.txn_delete_bitmap_cache().remove_unused_tablet_txn_info(txn_id, + tablet->tablet_id()); + return Status::OK(); + } else { + LOG(WARNING) << "failed to get tablet txn info. tablet_id=" << tablet->tablet_id() + << ", txn_id=" << txn_id << ", status=" << status; + } + } + std::shared_ptr stub; RETURN_IF_ERROR(MetaServiceProxy::get_client(&stub)); diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 587b1638a00ead..9bd15cd6e361cb 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -613,14 +613,24 @@ Status CloudTablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t tx RETURN_IF_ERROR(_engine.meta_mgr().update_tmp_rowset(*rowset_meta)); } + RowsetSharedPtr tmp_rowset; + DeleteBitmapPtr tmp_delete_bitmap; + RowsetIdUnorderedSet tmp_rowset_ids; + std::shared_ptr tmp_partial_update_info; + int64_t tmp_txn_expiration; + RETURN_IF_ERROR(_engine.txn_delete_bitmap_cache().get_tablet_txn_info( + txn_id, tablet_id(), &tmp_rowset, &tmp_delete_bitmap, &tmp_rowset_ids, + &tmp_txn_expiration, &tmp_partial_update_info)); DeleteBitmapPtr new_delete_bitmap = std::make_shared(tablet_id()); for (auto iter = delete_bitmap->delete_bitmap.begin(); iter != delete_bitmap->delete_bitmap.end(); ++iter) { // skip sentinel mark, which is used for delete bitmap correctness check if (std::get<1>(iter->first) != DeleteBitmap::INVALID_SEGMENT_ID) { - new_delete_bitmap->merge( - {std::get<0>(iter->first), std::get<1>(iter->first), cur_version}, - iter->second); + DeleteBitmap::BitmapKey key = {std::get<0>(iter->first), std::get<1>(iter->first), + cur_version}; + tmp_delete_bitmap->status_map.insert( + std::make_pair(key, DeleteBitmap::PublishStatus::FINALIZED)); + new_delete_bitmap->merge(key, iter->second); } } diff --git a/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp b/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp index 5a242f9af3a6c7..857b7b1ee5fd86 100644 --- a/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp +++ b/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp @@ -152,6 +152,10 @@ void CloudTxnDeleteBitmapCache::remove_expired_tablet_txn_info() { std::unique_lock wlock(_rwlock); while (!_expiration_txn.empty()) { auto iter = _expiration_txn.begin(); + if (_txn_map.find(iter->second) == _txn_map.end()) { + _expiration_txn.erase(iter); + continue; + } int64_t current_time = duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); @@ -174,6 +178,23 @@ void CloudTxnDeleteBitmapCache::remove_expired_tablet_txn_info() { } } +void CloudTxnDeleteBitmapCache::remove_unused_tablet_txn_info(TTransactionId transaction_id, + int64_t tablet_id) { + std::unique_lock wlock(_rwlock); + TxnKey txn_key(transaction_id, tablet_id); + auto txn_iter = _txn_map.find(txn_key); + if (txn_iter != _txn_map.end()) { + LOG_INFO("remove unused tablet txn info") + .tag("txn_id", txn_iter->first.txn_id) + .tag("tablt_id", txn_iter->first.tablet_id); + std::string key_str = std::to_string(txn_iter->first.txn_id) + "/" + + std::to_string(txn_iter->first.tablet_id); // Cache key container + CacheKey cache_key(key_str); + erase(cache_key); + _txn_map.erase(txn_key); + } +} + void CloudTxnDeleteBitmapCache::_clean_thread_callback() { do { remove_expired_tablet_txn_info(); diff --git a/be/src/cloud/cloud_txn_delete_bitmap_cache.h b/be/src/cloud/cloud_txn_delete_bitmap_cache.h index 3b1d1d1d85760d..5b2c92e750afed 100644 --- a/be/src/cloud/cloud_txn_delete_bitmap_cache.h +++ b/be/src/cloud/cloud_txn_delete_bitmap_cache.h @@ -53,6 +53,8 @@ class CloudTxnDeleteBitmapCache : public LRUCachePolicy { void remove_expired_tablet_txn_info(); + void remove_unused_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id); + private: void _clean_thread_callback(); diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index e9ae4314b13e06..579a360c3a384d 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -371,8 +371,10 @@ class DeleteBitmap { mutable std::shared_mutex lock; using SegmentId = uint32_t; using Version = uint64_t; + enum PublishStatus { NONE = 0, FINALIZED = 1 }; using BitmapKey = std::tuple; std::map delete_bitmap; // Ordered map + std::map status_map; constexpr static inline uint32_t INVALID_SEGMENT_ID = std::numeric_limits::max() - 1; constexpr static inline uint32_t ROWSET_SENTINEL_MARK = std::numeric_limits::max() - 1; From f056aee43c8b2ca3beb1f418924de21dbeadba04 Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Tue, 4 Jun 2024 16:04:32 +0800 Subject: [PATCH 2/4] edit --- be/src/cloud/cloud_meta_mgr.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index d1f8f3033efabd..c935ed8f547b28 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -573,8 +573,6 @@ Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_ iter != tmp_delete_bitmap->delete_bitmap.end(); ++iter) { DeleteBitmap::BitmapKey key = {std::get<0>(iter->first), std::get<1>(iter->first), old_max_version + 1}; - LOG(INFO) << "key " << std::get<0>(iter->first) << "," << std::get<1>(iter->first) - << "," << old_max_version + 1; auto it = tmp_delete_bitmap->status_map.find(key); if (it != tmp_delete_bitmap->status_map.end() && it->second == DeleteBitmap::PublishStatus::FINALIZED) { From 753f892d2c3b6eb6079dec2a4a27e7d40d149871 Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Thu, 13 Jun 2024 18:00:55 +0800 Subject: [PATCH 3/4] edit --- .../cloud_engine_calc_delete_bitmap_task.cpp | 14 ++- be/src/cloud/cloud_meta_mgr.cpp | 103 ++++++++++++------ be/src/cloud/cloud_meta_mgr.h | 4 + be/src/cloud/cloud_tablet.cpp | 22 ++-- .../cloud/cloud_txn_delete_bitmap_cache.cpp | 38 ++++++- be/src/cloud/cloud_txn_delete_bitmap_cache.h | 18 ++- be/src/olap/tablet_meta.h | 2 - be/src/olap/txn_manager.h | 2 + 8 files changed, 145 insertions(+), 58 deletions(-) diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp index b9269adf1a7a95..61fcc311434b36 100644 --- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp +++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp @@ -154,10 +154,11 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const { DeleteBitmapPtr delete_bitmap; RowsetIdUnorderedSet rowset_ids; std::shared_ptr partial_update_info; + std::shared_ptr publish_status; int64_t txn_expiration; Status status = _engine.txn_delete_bitmap_cache().get_tablet_txn_info( _transaction_id, _tablet_id, &rowset, &delete_bitmap, &rowset_ids, &txn_expiration, - &partial_update_info); + &partial_update_info, &publish_status); if (status != Status::OK()) { LOG(WARNING) << "failed to get tablet txn info. tablet_id=" << _tablet_id << ", txn_id=" << _transaction_id << ", status=" << status; @@ -172,8 +173,15 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const { txn_info.delete_bitmap = delete_bitmap; txn_info.rowset_ids = rowset_ids; txn_info.partial_update_info = partial_update_info; - status = CloudTablet::update_delete_bitmap(tablet, &txn_info, _transaction_id, txn_expiration); - auto update_delete_bitmap_time_us = MonotonicMicros() - t3; + txn_info.publish_status = publish_status; + auto update_delete_bitmap_time_us = 0; + if (txn_info.publish_status && (*(txn_info.publish_status) == PublishStatus::SUCCEED)) { + LOG(INFO) << "tablet=" << _tablet_id << ",txn=" << _transaction_id + << ",publish_status=SUCCEED,not need to recalculate and update delete_bitmap."; + } else { + status = CloudTablet::update_delete_bitmap(tablet, &txn_info, _transaction_id, txn_expiration); + update_delete_bitmap_time_us = MonotonicMicros() - t3; + } if (status != Status::OK()) { LOG(WARNING) << "failed to calculate delete bitmap. rowset_id=" << rowset->rowset_id() << ", tablet_id=" << _tablet_id << ", txn_id=" << _transaction_id diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index c935ed8f547b28..e50478d3d8ae58 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -548,46 +548,87 @@ Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet* tablet, bool warmup_delta_ } } -Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_max_version, - std::ranges::range auto&& rs_metas, - const TabletStatsPB& stats, const TabletIndexPB& idx, - DeleteBitmap* delete_bitmap) { - if (rs_metas.empty()) { - return Status::OK(); - } - - { - auto txn_id = rs_metas.rbegin()->txn_id(); - RowsetSharedPtr rowset; +bool CloudMetaMgr::sync_tablet_delete_bitmap_by_cache(Tablet* tablet, int64_t old_max_version, + std::ranges::range auto&& rs_metas, + DeleteBitmap* delete_bitmap) { + std::set txn_processed; + for (auto& rs_meta : rs_metas) { + auto txn_id = rs_meta.txn_id(); + if (txn_processed.find(txn_id) != txn_processed.end()) { + continue; + } + txn_processed.insert(txn_id); DeleteBitmapPtr tmp_delete_bitmap; - RowsetIdUnorderedSet rowset_ids; - std::shared_ptr partial_update_info; - int64_t txn_expiration; + RowsetIdUnorderedSet tmp_rowset_ids; + std::shared_ptr publish_status = + std::make_shared(PublishStatus::INIT); + // Status status = StorageEngine::instance()->delete_bitmap_txn_manager()->get_delete_bitmap( + // txn_id, tablet->tablet_id(), &tmp_delete_bitmap, &tmp_rowset_ids, &publish_status); CloudStorageEngine& engine = ExecEnv::GetInstance()->storage_engine().to_cloud(); - Status status = engine.txn_delete_bitmap_cache().get_tablet_txn_info( - txn_id, tablet->tablet_id(), &rowset, &tmp_delete_bitmap, &rowset_ids, - &txn_expiration, &partial_update_info); - if (status.ok()) { - DeleteBitmapPtr new_delete_bitmap = std::make_shared(tablet->tablet_id()); - for (auto iter = tmp_delete_bitmap->delete_bitmap.begin(); - iter != tmp_delete_bitmap->delete_bitmap.end(); ++iter) { - DeleteBitmap::BitmapKey key = {std::get<0>(iter->first), std::get<1>(iter->first), - old_max_version + 1}; - auto it = tmp_delete_bitmap->status_map.find(key); - if (it != tmp_delete_bitmap->status_map.end() && - it->second == DeleteBitmap::PublishStatus::FINALIZED) { - new_delete_bitmap->merge(key, iter->second); - } - } - *delete_bitmap = *new_delete_bitmap; + Status status = engine.txn_delete_bitmap_cache().get_delete_bitmap( + txn_id, tablet->tablet_id(), &tmp_delete_bitmap, &tmp_rowset_ids, &publish_status); + if (status.ok() && *(publish_status.get()) == PublishStatus::SUCCEED) { + delete_bitmap->merge(*tmp_delete_bitmap); engine.txn_delete_bitmap_cache().remove_unused_tablet_txn_info(txn_id, tablet->tablet_id()); - return Status::OK(); } else { LOG(WARNING) << "failed to get tablet txn info. tablet_id=" << tablet->tablet_id() << ", txn_id=" << txn_id << ", status=" << status; + return false; } } + return true; +} + +Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_max_version, + std::ranges::range auto&& rs_metas, + const TabletStatsPB& stats, const TabletIndexPB& idx, + DeleteBitmap* delete_bitmap) { + if (rs_metas.empty()) { + return Status::OK(); + } + +// { +// auto txn_id = rs_metas.rbegin()->txn_id(); +// RowsetSharedPtr rowset; +// DeleteBitmapPtr tmp_delete_bitmap; +// RowsetIdUnorderedSet rowset_ids; +// std::shared_ptr partial_update_info; +// int64_t txn_expiration; +// CloudStorageEngine& engine = ExecEnv::GetInstance()->storage_engine().to_cloud(); +// Status status = engine.txn_delete_bitmap_cache().get_tablet_txn_info( +// txn_id, tablet->tablet_id(), &rowset, &tmp_delete_bitmap, &rowset_ids, +// &txn_expiration, &partial_update_info); +// if (status.ok()) { +// DeleteBitmapPtr new_delete_bitmap = std::make_shared(tablet->tablet_id()); +// for (auto iter = tmp_delete_bitmap->delete_bitmap.begin(); +// iter != tmp_delete_bitmap->delete_bitmap.end(); ++iter) { +// DeleteBitmap::BitmapKey key = {std::get<0>(iter->first), std::get<1>(iter->first), +// old_max_version + 1}; +// auto it = tmp_delete_bitmap->status_map.find(key); +// if (it != tmp_delete_bitmap->status_map.end() && +// it->second == DeleteBitmap::PublishStatus::FINALIZED) { +// new_delete_bitmap->merge(key, iter->second); +// } +// } +// *delete_bitmap = *new_delete_bitmap; +// engine.txn_delete_bitmap_cache().remove_unused_tablet_txn_info(txn_id, +// tablet->tablet_id()); +// return Status::OK(); +// } else { +// LOG(WARNING) << "failed to get tablet txn info. tablet_id=" << tablet->tablet_id() +// << ", txn_id=" << txn_id << ", status=" << status; +// } +// } + + if (sync_tablet_delete_bitmap_by_cache(tablet, old_max_version, rs_metas, delete_bitmap)) { + return Status::OK(); + } else { + LOG(WARNING) << "failed to sync delete bitmap by txn info. tablet_id=" + << tablet->tablet_id(); + DeleteBitmapPtr new_delete_bitmap = std::make_shared(tablet->tablet_id()); + *delete_bitmap = *new_delete_bitmap; + } std::shared_ptr stub; RETURN_IF_ERROR(MetaServiceProxy::get_client(&stub)); diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h index 0164cf57517ed5..32e5969c0e1167 100644 --- a/be/src/cloud/cloud_meta_mgr.h +++ b/be/src/cloud/cloud_meta_mgr.h @@ -91,6 +91,10 @@ class CloudMetaMgr { Status get_delete_bitmap_update_lock(const CloudTablet& tablet, int64_t lock_id, int64_t initiator); + bool sync_tablet_delete_bitmap_by_cache(Tablet* tablet, int64_t old_max_version, + std::ranges::range auto&& rs_metas, + DeleteBitmap* delete_bitmap); + private: Status sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_max_version, std::ranges::range auto&& rs_metas, const TabletStatsPB& stats, diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 9bd15cd6e361cb..c09d57e500e573 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -604,8 +604,8 @@ Status CloudTablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t tx RowsetSharedPtr rowset = txn_info->rowset; int64_t cur_version = rowset->start_version(); // update delete bitmap info, in order to avoid recalculation when trying again - _engine.txn_delete_bitmap_cache().update_tablet_txn_info(txn_id, tablet_id(), delete_bitmap, - cur_rowset_ids); + _engine.txn_delete_bitmap_cache().update_tablet_txn_info( + txn_id, tablet_id(), delete_bitmap, cur_rowset_ids, PublishStatus::PREPARE); if (txn_info->partial_update_info && txn_info->partial_update_info->is_partial_update && rowset_writer->num_rows() > 0) { @@ -613,29 +613,21 @@ Status CloudTablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t tx RETURN_IF_ERROR(_engine.meta_mgr().update_tmp_rowset(*rowset_meta)); } - RowsetSharedPtr tmp_rowset; - DeleteBitmapPtr tmp_delete_bitmap; - RowsetIdUnorderedSet tmp_rowset_ids; - std::shared_ptr tmp_partial_update_info; - int64_t tmp_txn_expiration; - RETURN_IF_ERROR(_engine.txn_delete_bitmap_cache().get_tablet_txn_info( - txn_id, tablet_id(), &tmp_rowset, &tmp_delete_bitmap, &tmp_rowset_ids, - &tmp_txn_expiration, &tmp_partial_update_info)); DeleteBitmapPtr new_delete_bitmap = std::make_shared(tablet_id()); for (auto iter = delete_bitmap->delete_bitmap.begin(); iter != delete_bitmap->delete_bitmap.end(); ++iter) { // skip sentinel mark, which is used for delete bitmap correctness check if (std::get<1>(iter->first) != DeleteBitmap::INVALID_SEGMENT_ID) { - DeleteBitmap::BitmapKey key = {std::get<0>(iter->first), std::get<1>(iter->first), - cur_version}; - tmp_delete_bitmap->status_map.insert( - std::make_pair(key, DeleteBitmap::PublishStatus::FINALIZED)); - new_delete_bitmap->merge(key, iter->second); + new_delete_bitmap->merge( + {std::get<0>(iter->first), std::get<1>(iter->first), cur_version}, + iter->second); } } RETURN_IF_ERROR(_engine.meta_mgr().update_delete_bitmap( *this, txn_id, COMPACTION_DELETE_BITMAP_LOCK_ID, new_delete_bitmap.get())); + _engine.txn_delete_bitmap_cache().update_tablet_txn_info( + txn_id, tablet_id(), new_delete_bitmap, cur_rowset_ids, PublishStatus::SUCCEED); return Status::OK(); } diff --git a/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp b/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp index 857b7b1ee5fd86..707fe17a0851d4 100644 --- a/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp +++ b/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp @@ -53,7 +53,8 @@ Status CloudTxnDeleteBitmapCache::init() { Status CloudTxnDeleteBitmapCache::get_tablet_txn_info( TTransactionId transaction_id, int64_t tablet_id, RowsetSharedPtr* rowset, DeleteBitmapPtr* delete_bitmap, RowsetIdUnorderedSet* rowset_ids, int64_t* txn_expiration, - std::shared_ptr* partial_update_info) { + std::shared_ptr* partial_update_info, + std::shared_ptr* publish_status) { { std::shared_lock rlock(_rwlock); TxnKey key(transaction_id, tablet_id); @@ -66,6 +67,27 @@ Status CloudTxnDeleteBitmapCache::get_tablet_txn_info( *rowset = iter->second.rowset; *txn_expiration = iter->second.txn_expiration; *partial_update_info = iter->second.partial_update_info; + *publish_status = iter->second.publish_status; + } + RETURN_IF_ERROR( + get_delete_bitmap(transaction_id, tablet_id, delete_bitmap, rowset_ids, nullptr)); + return Status::OK(); +} + +Status CloudTxnDeleteBitmapCache::get_delete_bitmap(TTransactionId transaction_id, int64_t tablet_id, + DeleteBitmapPtr* delete_bitmap, + RowsetIdUnorderedSet* rowset_ids, + std::shared_ptr* publish_status) { + if (publish_status) { + std::shared_lock rlock(_rwlock); + TxnKey txn_key(transaction_id, tablet_id); + auto iter = _txn_map.find(txn_key); + if (iter == _txn_map.end()) { + return Status::Error( + "not found txn info, tablet_id={}, transaction_id={}", tablet_id, + transaction_id); + } + *publish_status = iter->second.publish_status; } std::string key_str = fmt::format("{}/{}", transaction_id, tablet_id); CacheKey key(key_str); @@ -103,7 +125,10 @@ void CloudTxnDeleteBitmapCache::set_tablet_txn_info( { std::unique_lock wlock(_rwlock); TxnKey txn_key(transaction_id, tablet_id); - _txn_map[txn_key] = TxnVal(rowset, txn_expiration, std::move(partial_update_info)); + std::shared_ptr publish_status = + std::make_shared(PublishStatus::INIT); + _txn_map[txn_key] = TxnVal(rowset, txn_expiration, std::move(partial_update_info), + std::move(publish_status)); _expiration_txn.emplace(txn_expiration, txn_key); } std::string key_str = fmt::format("{}/{}", transaction_id, tablet_id); @@ -128,7 +153,14 @@ void CloudTxnDeleteBitmapCache::set_tablet_txn_info( void CloudTxnDeleteBitmapCache::update_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id, DeleteBitmapPtr delete_bitmap, - const RowsetIdUnorderedSet& rowset_ids) { + const RowsetIdUnorderedSet& rowset_ids, + PublishStatus publish_status) { + { + std::unique_lock wlock(_rwlock); + TxnKey txn_key(transaction_id, tablet_id); + CHECK(_txn_map.count(txn_key) > 0); + *(_txn_map[txn_key].publish_status.get()) = publish_status; + } std::string key_str = fmt::format("{}/{}", transaction_id, tablet_id); CacheKey key(key_str); diff --git a/be/src/cloud/cloud_txn_delete_bitmap_cache.h b/be/src/cloud/cloud_txn_delete_bitmap_cache.h index 5b2c92e750afed..71d5123f34dcc8 100644 --- a/be/src/cloud/cloud_txn_delete_bitmap_cache.h +++ b/be/src/cloud/cloud_txn_delete_bitmap_cache.h @@ -24,6 +24,7 @@ #include "olap/partial_update_info.h" #include "olap/rowset/rowset.h" #include "olap/tablet_meta.h" +#include "olap/txn_manager.h" #include "util/countdown_latch.h" namespace doris { @@ -40,7 +41,8 @@ class CloudTxnDeleteBitmapCache : public LRUCachePolicy { Status get_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id, RowsetSharedPtr* rowset, DeleteBitmapPtr* delete_bitmap, RowsetIdUnorderedSet* rowset_ids, int64_t* txn_expiration, - std::shared_ptr* partial_update_info); + std::shared_ptr* partial_update_info, + std::shared_ptr* publish_status); void set_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id, DeleteBitmapPtr delete_bitmap, const RowsetIdUnorderedSet& rowset_ids, @@ -49,12 +51,17 @@ class CloudTxnDeleteBitmapCache : public LRUCachePolicy { void update_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id, DeleteBitmapPtr delete_bitmap, - const RowsetIdUnorderedSet& rowset_ids); + const RowsetIdUnorderedSet& rowset_ids, + PublishStatus publish_status); void remove_expired_tablet_txn_info(); void remove_unused_tablet_txn_info(TTransactionId transaction_id, int64_t tablet_id); + Status get_delete_bitmap(TTransactionId transaction_id, int64_t tablet_id, + DeleteBitmapPtr* delete_bitmap, RowsetIdUnorderedSet* rowset_ids, + std::shared_ptr* publish_status); + private: void _clean_thread_callback(); @@ -82,12 +89,15 @@ class CloudTxnDeleteBitmapCache : public LRUCachePolicy { RowsetSharedPtr rowset; int64_t txn_expiration; std::shared_ptr partial_update_info; + std::shared_ptr publish_status = nullptr; TxnVal() : txn_expiration(0) {}; TxnVal(RowsetSharedPtr rowset_, int64_t txn_expiration_, - std::shared_ptr partial_update_info_) + std::shared_ptr partial_update_info_, + std::shared_ptr publish_status_) : rowset(std::move(rowset_)), txn_expiration(txn_expiration_), - partial_update_info(std::move(partial_update_info_)) {} + partial_update_info(std::move(partial_update_info_)), + publish_status(std::move(publish_status_)) {} }; std::map _txn_map; diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 579a360c3a384d..e9ae4314b13e06 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -371,10 +371,8 @@ class DeleteBitmap { mutable std::shared_mutex lock; using SegmentId = uint32_t; using Version = uint64_t; - enum PublishStatus { NONE = 0, FINALIZED = 1 }; using BitmapKey = std::tuple; std::map delete_bitmap; // Ordered map - std::map status_map; constexpr static inline uint32_t INVALID_SEGMENT_ID = std::numeric_limits::max() - 1; constexpr static inline uint32_t ROWSET_SENTINEL_MARK = std::numeric_limits::max() - 1; diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h index 724255bccafa53..9fd528b91a7e3d 100644 --- a/be/src/olap/txn_manager.h +++ b/be/src/olap/txn_manager.h @@ -61,6 +61,7 @@ enum class TxnState { ABORTED = 4, DELETED = 5, }; +enum class PublishStatus { INIT = 0, PREPARE = 1, SUCCEED = 2 }; struct TabletTxnInfo { PUniqueId load_id; @@ -73,6 +74,7 @@ struct TabletTxnInfo { int64_t creation_time; bool ingest {false}; std::shared_ptr partial_update_info; + std::shared_ptr publish_status; TxnState state {TxnState::PREPARED}; TabletTxnInfo() = default; From 6f2f281fe96d9979739ead9055ccb14eaa7c34f2 Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Fri, 14 Jun 2024 13:22:21 +0800 Subject: [PATCH 4/4] edit --- .../cloud_engine_calc_delete_bitmap_task.cpp | 5 ++- be/src/cloud/cloud_meta_mgr.cpp | 37 +------------------ be/src/cloud/cloud_meta_mgr.h | 4 +- .../cloud/cloud_txn_delete_bitmap_cache.cpp | 7 ++-- 4 files changed, 9 insertions(+), 44 deletions(-) diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp index 61fcc311434b36..723bcdb48faffd 100644 --- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp +++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp @@ -176,10 +176,11 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const { txn_info.publish_status = publish_status; auto update_delete_bitmap_time_us = 0; if (txn_info.publish_status && (*(txn_info.publish_status) == PublishStatus::SUCCEED)) { - LOG(INFO) << "tablet=" << _tablet_id << ",txn=" << _transaction_id + LOG(INFO) << "tablet=" << _tablet_id << ",txn=" << _transaction_id << ",publish_status=SUCCEED,not need to recalculate and update delete_bitmap."; } else { - status = CloudTablet::update_delete_bitmap(tablet, &txn_info, _transaction_id, txn_expiration); + status = CloudTablet::update_delete_bitmap(tablet, &txn_info, _transaction_id, + txn_expiration); update_delete_bitmap_time_us = MonotonicMicros() - t3; } if (status != Status::OK()) { diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index e50478d3d8ae58..f0a377cba67dd1 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -548,7 +548,7 @@ Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet* tablet, bool warmup_delta_ } } -bool CloudMetaMgr::sync_tablet_delete_bitmap_by_cache(Tablet* tablet, int64_t old_max_version, +bool CloudMetaMgr::sync_tablet_delete_bitmap_by_cache(CloudTablet* tablet, int64_t old_max_version, std::ranges::range auto&& rs_metas, DeleteBitmap* delete_bitmap) { std::set txn_processed; @@ -562,8 +562,6 @@ bool CloudMetaMgr::sync_tablet_delete_bitmap_by_cache(Tablet* tablet, int64_t ol RowsetIdUnorderedSet tmp_rowset_ids; std::shared_ptr publish_status = std::make_shared(PublishStatus::INIT); - // Status status = StorageEngine::instance()->delete_bitmap_txn_manager()->get_delete_bitmap( - // txn_id, tablet->tablet_id(), &tmp_delete_bitmap, &tmp_rowset_ids, &publish_status); CloudStorageEngine& engine = ExecEnv::GetInstance()->storage_engine().to_cloud(); Status status = engine.txn_delete_bitmap_cache().get_delete_bitmap( txn_id, tablet->tablet_id(), &tmp_delete_bitmap, &tmp_rowset_ids, &publish_status); @@ -588,39 +586,6 @@ Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_ return Status::OK(); } -// { -// auto txn_id = rs_metas.rbegin()->txn_id(); -// RowsetSharedPtr rowset; -// DeleteBitmapPtr tmp_delete_bitmap; -// RowsetIdUnorderedSet rowset_ids; -// std::shared_ptr partial_update_info; -// int64_t txn_expiration; -// CloudStorageEngine& engine = ExecEnv::GetInstance()->storage_engine().to_cloud(); -// Status status = engine.txn_delete_bitmap_cache().get_tablet_txn_info( -// txn_id, tablet->tablet_id(), &rowset, &tmp_delete_bitmap, &rowset_ids, -// &txn_expiration, &partial_update_info); -// if (status.ok()) { -// DeleteBitmapPtr new_delete_bitmap = std::make_shared(tablet->tablet_id()); -// for (auto iter = tmp_delete_bitmap->delete_bitmap.begin(); -// iter != tmp_delete_bitmap->delete_bitmap.end(); ++iter) { -// DeleteBitmap::BitmapKey key = {std::get<0>(iter->first), std::get<1>(iter->first), -// old_max_version + 1}; -// auto it = tmp_delete_bitmap->status_map.find(key); -// if (it != tmp_delete_bitmap->status_map.end() && -// it->second == DeleteBitmap::PublishStatus::FINALIZED) { -// new_delete_bitmap->merge(key, iter->second); -// } -// } -// *delete_bitmap = *new_delete_bitmap; -// engine.txn_delete_bitmap_cache().remove_unused_tablet_txn_info(txn_id, -// tablet->tablet_id()); -// return Status::OK(); -// } else { -// LOG(WARNING) << "failed to get tablet txn info. tablet_id=" << tablet->tablet_id() -// << ", txn_id=" << txn_id << ", status=" << status; -// } -// } - if (sync_tablet_delete_bitmap_by_cache(tablet, old_max_version, rs_metas, delete_bitmap)) { return Status::OK(); } else { diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h index 32e5969c0e1167..6f6cc9c26b47b4 100644 --- a/be/src/cloud/cloud_meta_mgr.h +++ b/be/src/cloud/cloud_meta_mgr.h @@ -91,11 +91,11 @@ class CloudMetaMgr { Status get_delete_bitmap_update_lock(const CloudTablet& tablet, int64_t lock_id, int64_t initiator); - bool sync_tablet_delete_bitmap_by_cache(Tablet* tablet, int64_t old_max_version, +private: + bool sync_tablet_delete_bitmap_by_cache(CloudTablet* tablet, int64_t old_max_version, std::ranges::range auto&& rs_metas, DeleteBitmap* delete_bitmap); -private: Status sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_max_version, std::ranges::range auto&& rs_metas, const TabletStatsPB& stats, const TabletIndexPB& idx, DeleteBitmap* delete_bitmap); diff --git a/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp b/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp index 707fe17a0851d4..64faf915e612e2 100644 --- a/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp +++ b/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp @@ -74,10 +74,9 @@ Status CloudTxnDeleteBitmapCache::get_tablet_txn_info( return Status::OK(); } -Status CloudTxnDeleteBitmapCache::get_delete_bitmap(TTransactionId transaction_id, int64_t tablet_id, - DeleteBitmapPtr* delete_bitmap, - RowsetIdUnorderedSet* rowset_ids, - std::shared_ptr* publish_status) { +Status CloudTxnDeleteBitmapCache::get_delete_bitmap( + TTransactionId transaction_id, int64_t tablet_id, DeleteBitmapPtr* delete_bitmap, + RowsetIdUnorderedSet* rowset_ids, std::shared_ptr* publish_status) { if (publish_status) { std::shared_lock rlock(_rwlock); TxnKey txn_key(transaction_id, tablet_id);