From 96fa094e3551336ab3ecbfecdd3077faa96d2777 Mon Sep 17 00:00:00 2001 From: liulijia Date: Wed, 11 Dec 2019 17:21:42 +0800 Subject: [PATCH 1/2] Fix #2419 * Add a flag in RowsetMeta to record whether it has been deleted from rowset meta. --- be/src/olap/rowset/rowset_meta.h | 14 ++++++++++++++ be/src/olap/storage_engine.cpp | 2 +- be/src/olap/tablet.cpp | 7 ++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index dfb80c79c0da00..a3259d667d91c3 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -296,6 +296,19 @@ class RowsetMeta { return has_version() && _rowset_meta_pb.start_version() == _rowset_meta_pb.end_version(); } + // Some time, we may check if this rowset is in rowset meta manager's meta by using RowsetMetaManager::check_rowset_meta. + // But, this check behavior may cost a lot of time when it is frequent. + // If we explicitly remove this rowset from rowset meta manager's meta, we can set _is_removed_from_rowset_meta to true, + // And next time when we want to check if this rowset is in rowset mata manager's meta, we can + // check is_remove_from_rowset_meta() first. + void set_remove_from_rowset_meta() { + _is_removed_from_rowset_meta = true; + } + + bool is_remove_from_rowset_meta() { + return _is_removed_from_rowset_meta; + } + private: friend class AlphaRowsetMeta; bool _deserialize_from_pb(const std::string& value) { @@ -332,6 +345,7 @@ class RowsetMeta { private: RowsetMetaPB _rowset_meta_pb; RowsetId _rowset_id; + bool _is_removed_from_rowset_meta = false; }; } // namespace doris diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index bfb7ac92f12355..4a2b2a770dfe25 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -1021,7 +1021,7 @@ void* StorageEngine::_tablet_checkpoint_callback(void* arg) { #endif LOG(INFO) << "try to start tablet meta checkpoint thread!"; while (true) { - LOG(INFO) << "begin to do tablet meta checkpoint"; + LOG(INFO) << "begin to do tablet meta checkpoint:" << ((DataDir*)arg)->path(); int64_t start_time = UnixMillis(); _tablet_manager->do_tablet_meta_checkpoint((DataDir*)arg); int64_t used_time = (UnixMillis() - start_time) / 1000; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 998b61d3d68cdb..de9f63ffe56d87 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1065,12 +1065,17 @@ OLAPStatus Tablet::do_tablet_meta_checkpoint() { RETURN_NOT_OK(save_meta()); // if save meta successfully, then should remove the rowset meta existing in tablet // meta from rowset meta store - for (auto& rs_meta : _tablet_meta->all_rs_metas()) { + for (auto& rs_meta : _tablet_meta->all_rs_metas()) { + // If we delete it from rowset manager's meta explicitly in previous checkpoint, just skip. + if(rs_meta->is_remove_from_rowset_meta()) { + continue; + } if (RowsetMetaManager::check_rowset_meta(_data_dir->get_meta(), tablet_uid(), rs_meta->rowset_id())) { RowsetMetaManager::remove(_data_dir->get_meta(), tablet_uid(), rs_meta->rowset_id()); LOG(INFO) << "remove rowset id from meta store because it is already persistent with tablet meta" << ", rowset_id=" << rs_meta->rowset_id(); } + rs_meta->set_remove_from_rowset_meta(); } _newly_created_rowset_num = 0; _last_checkpoint_time = UnixMillis(); From b79384f43edccc0c021ba718926b1bff322050f2 Mon Sep 17 00:00:00 2001 From: Lijia Liu Date: Thu, 12 Dec 2019 14:39:26 +0800 Subject: [PATCH 2/2] Update is_remove_from_rowset_meta return const Co-Authored-By: ZHAO Chun --- be/src/olap/rowset/rowset_meta.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index a3259d667d91c3..a4ab71cac2bdb3 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -305,7 +305,7 @@ class RowsetMeta { _is_removed_from_rowset_meta = true; } - bool is_remove_from_rowset_meta() { + bool is_remove_from_rowset_meta() const { return _is_removed_from_rowset_meta; }