Fix BE do_tablet_meta_checkpoint retain _meta_lock for a long time

be/src/olap/rowset/rowset_meta.h

@@ -296,6 +296,19 @@ class RowsetMeta {
         return  has_version() && _rowset_meta_pb.start_version() == _rowset_meta_pb.end_version();
     }
 
+    // Some time, we may check if this rowset is in rowset meta manager's meta by using RowsetMetaManager::check_rowset_meta.
+    // But, this check behavior may cost a lot of time when it is frequent.
+    // If we explicitly remove this rowset from rowset meta manager's meta, we can set _is_removed_from_rowset_meta to true,
+    // And next time when we want to check if this rowset is in rowset mata manager's meta, we can
+    // check is_remove_from_rowset_meta() first.
+    void set_remove_from_rowset_meta() {
+        _is_removed_from_rowset_meta = true;
+    }
+
+    bool is_remove_from_rowset_meta() const {
+        return _is_removed_from_rowset_meta;
+    }
+
 private:
     friend class AlphaRowsetMeta;
     bool _deserialize_from_pb(const std::string& value) {
@@ -332,6 +345,7 @@ class RowsetMeta {
 private:
     RowsetMetaPB _rowset_meta_pb;
     RowsetId _rowset_id;
+    bool _is_removed_from_rowset_meta = false;
 };
 
 } // namespace doris

be/src/olap/storage_engine.cpp

@@ -1021,7 +1021,7 @@ void* StorageEngine::_tablet_checkpoint_callback(void* arg) {
 #endif
     LOG(INFO) << "try to start tablet meta checkpoint thread!";
     while (true) {
-        LOG(INFO) << "begin to do tablet meta checkpoint";
+        LOG(INFO) << "begin to do tablet meta checkpoint:" << ((DataDir*)arg)->path();
         int64_t start_time = UnixMillis();
         _tablet_manager->do_tablet_meta_checkpoint((DataDir*)arg);
         int64_t used_time = (UnixMillis() - start_time) / 1000;

be/src/olap/tablet.cpp

@@ -1065,12 +1065,17 @@ OLAPStatus Tablet::do_tablet_meta_checkpoint() {
     RETURN_NOT_OK(save_meta());
     // if save meta successfully, then should remove the rowset meta existing in tablet
     // meta from rowset meta store
-    for (auto& rs_meta :  _tablet_meta->all_rs_metas()) {
+    for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
+        // If we delete it from rowset manager's meta explicitly in previous checkpoint, just skip.
+        if(rs_meta->is_remove_from_rowset_meta()) {
+            continue;
+        }
         if (RowsetMetaManager::check_rowset_meta(_data_dir->get_meta(), tablet_uid(), rs_meta->rowset_id())) {
             RowsetMetaManager::remove(_data_dir->get_meta(), tablet_uid(), rs_meta->rowset_id());
             LOG(INFO) << "remove rowset id from meta store because it is already persistent with tablet meta"
                        << ", rowset_id=" << rs_meta->rowset_id();
         }
+        rs_meta->set_remove_from_rowset_meta();
     }
     _newly_created_rowset_num = 0;
     _last_checkpoint_time = UnixMillis();