[Opt](checker) Add checker for delete bitmaps

cloud/src/common/bvars.cpp

@@ -198,3 +198,10 @@ BvarStatusWithTag<long> g_bvar_checker_instance_volume("checker", "instance_volu
 BvarStatusWithTag<long> g_bvar_inverted_checker_num_scanned("checker", "num_inverted_scanned");
 BvarStatusWithTag<long> g_bvar_inverted_checker_num_check_failed("checker",
                                                                  "num_inverted_check_failed");
+
+BvarStatusWithTag<int64_t> g_bvar_inverted_checker_leaked_delete_bitmaps("checker",
+                                                                         "leaked_delete_bitmaps");
+BvarStatusWithTag<int64_t> g_bvar_inverted_checker_abnormal_delete_bitmaps(
+        "checker", "abnormal_delete_bitmaps");
+BvarStatusWithTag<int64_t> g_bvar_inverted_checker_delete_bitmaps_scanned(
+        "checker", "delete_bitmap_keys_scanned");

cloud/src/common/bvars.h

@@ -250,3 +250,7 @@ extern BvarStatusWithTag<long> g_bvar_checker_last_success_time_ms;
 extern BvarStatusWithTag<long> g_bvar_checker_instance_volume;
 extern BvarStatusWithTag<long> g_bvar_inverted_checker_num_scanned;
 extern BvarStatusWithTag<long> g_bvar_inverted_checker_num_check_failed;
+
+extern BvarStatusWithTag<int64_t> g_bvar_inverted_checker_leaked_delete_bitmaps;
+extern BvarStatusWithTag<int64_t> g_bvar_inverted_checker_abnormal_delete_bitmaps;
+extern BvarStatusWithTag<int64_t> g_bvar_inverted_checker_delete_bitmaps_scanned;

cloud/src/common/config.h

@@ -72,6 +72,11 @@ CONF_Bool(enable_checker, "false");
 CONF_Int32(recycle_pool_parallelism, "40");
 // Currently only used for recycler test
 CONF_Bool(enable_inverted_check, "false");
+// Currently only used for recycler test
+CONF_Bool(enable_delete_bitmap_inverted_check, "false");
+// checks if https://github.com/apache/doris/pull/40204 works as expected
+CONF_Bool(enable_delete_bitmap_storage_optimize_check, "false");
+CONF_mInt64(delete_bitmap_storage_optimize_check_version_gap, "1000");
 // interval for scanning instances to do checks and inspections
 CONF_mInt32(scan_instances_interval_seconds, "60"); // 1min
 // interval for check object

cloud/src/recycler/checker.h

@@ -23,13 +23,18 @@
 #include <atomic>
 #include <condition_variable>
 #include <deque>
+#include <functional>
 #include <thread>
 #include <unordered_map>
 #include <unordered_set>
 
 #include "recycler/storage_vault_accessor.h"
 #include "recycler/white_black_list.h"
 
+namespace doris {
+class RowsetMetaCloudPB;
+} // namespace doris
+
 namespace doris::cloud {
 class StorageVaultAccessor;
 class InstanceChecker;
@@ -86,6 +91,18 @@ class InstanceChecker {
     // Return 1 if data loss is identified.
     // Return negative if a temporary error occurred during the check process.
     int do_check();
+
+    // Return 0 if success.
+    // Return 1 if delete bitmap leak is identified.
+    // Return negative if a temporary error occurred during the check process.
+    int do_delete_bitmap_inverted_check();
+
+    // checks if https://github.com/apache/doris/pull/40204 works as expected
+    // the stale delete bitmap will be cleared in MS when BE delete expired stale rowsets
+    // NOTE: stale rowsets will be lost after BE restarts, so there may be some stale delete bitmaps
+    // which will not be cleared.
+    int do_delete_bitmap_storage_optimize_check();
+
     // If there are multiple buckets, return the minimum lifecycle; if there are no buckets (i.e.
     // all accessors are HdfsAccessor), return INT64_MAX.
     // Return 0 if success, otherwise error
@@ -100,6 +117,17 @@ class InstanceChecker {
     // returns 0 for success otherwise error
     int init_storage_vault_accessors(const InstanceInfoPB& instance);
 
+    int traverse_mow_tablet(const std::function<int(int64_t)>& check_func);
+    int traverse_rowset_delete_bitmaps(
+            int64_t tablet_id, std::string rowset_id,
+            const std::function<int(int64_t, std::string_view, int64_t, int64_t)>& callback);
+    int collect_tablet_rowsets(
+            int64_t tablet_id,
+            const std::function<void(const doris::RowsetMetaCloudPB&)>& collect_cb);
+    int traverse_delete_bitmaps(const std::function<int(int64_t)>& check_func);
+
+    int check_delete_bitmap_storage_optimize(int64_t tablet_id);
+
     std::atomic_bool stopped_ {false};
     std::shared_ptr<TxnKv> txn_kv_;
     std::string instance_id_;

cloud/src/recycler/util.cpp

@@ -23,6 +23,7 @@
 
 #include "common/util.h"
 #include "meta-service/keys.h"
+#include "meta-service/meta_service_schema.h"
 #include "meta-service/txn_kv.h"
 #include "meta-service/txn_kv_error.h"
 
@@ -233,4 +234,69 @@ int lease_instance_recycle_job(TxnKv* txn_kv, std::string_view key, const std::s
     return 0;
 }
 
+int get_tablet_idx(TxnKv* txn_kv, const std::string& instance_id, int64_t tablet_id,
+                   TabletIndexPB& tablet_idx) {
+    std::unique_ptr<Transaction> txn;
+    TxnErrorCode err = txn_kv->create_txn(&txn);
+    if (err != TxnErrorCode::TXN_OK) {
+        LOG(WARNING) << "failed to create txn";
+        return -1;
+    }
+
+    std::string key, val;
+    meta_tablet_idx_key({instance_id, tablet_id}, &key);
+    err = txn->get(key, &val);
+    if (err != TxnErrorCode::TXN_OK) {
+        LOG(WARNING) << fmt::format("failed to get tablet_idx, err={} tablet_id={} key={}", err,
+                                    tablet_id, hex(key));
+        return -1;
+    }
+    if (!tablet_idx.ParseFromString(val)) [[unlikely]] {
+        LOG(WARNING) << fmt::format("malformed tablet index value, tablet_id={} key={}", tablet_id,
+                                    hex(key));
+        return -1;
+    }
+    if (tablet_id != tablet_idx.tablet_id()) [[unlikely]] {
+        LOG(WARNING) << "unexpected error given_tablet_id=" << tablet_id
+                     << " idx_pb_tablet_id=" << tablet_idx.tablet_id() << " key=" << hex(key);
+        return -1;
+    }
+    return 0;
+}
+
+int get_tablet_meta(TxnKv* txn_kv, const std::string& instance_id, int64_t tablet_id,
+                    TabletMetaCloudPB& tablet_meta) {
+    TabletIndexPB tablet_idx;
+    int ret = get_tablet_idx(txn_kv, instance_id, tablet_id, tablet_idx);
+    if (ret < 0) {
+        return ret;
+    }
+
+    std::unique_ptr<Transaction> txn;
+    TxnErrorCode err = txn_kv->create_txn(&txn);
+    if (err != TxnErrorCode::TXN_OK) {
+        LOG(WARNING) << "failed to create txn";
+        return -1;
+    }
+
+    std::string key, val;
+    meta_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(),
+                     tablet_idx.partition_id(), tablet_id},
+                    &key);
+    err = txn->get(key, &val);
+    if (err != TxnErrorCode::TXN_OK) {
+        LOG(WARNING) << fmt::format(
+                "failed to get tablet, err={}, table_id={}, index_id={}, partition_id={}, "
+                "tablet_id={} key={}",
+                err, tablet_idx.table_id(), tablet_idx.index_id(), tablet_idx.partition_id(),
+                tablet_id, hex(key));
+        return -1;
+    }
+    if (!tablet_meta.ParseFromString(val)) [[unlikely]] {
+        LOG(WARNING) << fmt::format("malformed tablet meta, tablet_id={} key={}", tablet_id,
+                                    hex(key));
+        return -1;
+    }
+    return 0;
+}
 } // namespace doris::cloud

cloud/src/recycler/util.h

@@ -85,4 +85,9 @@ inline std::string tablet_path_prefix(int64_t tablet_id) {
     return fmt::format("data/{}/", tablet_id);
 }
 
+int get_tablet_idx(TxnKv* txn_kv, const std::string& instance_id, int64_t tablet_id,
+                   TabletIndexPB& tablet_idx);
+
+int get_tablet_meta(TxnKv* txn_kv, const std::string& instance_id, int64_t tablet_id,
+                    TabletMetaCloudPB& tablet_meta);
 } // namespace doris::cloud