Skip to content

[opt](cloud-mow) add metrics for get/update delete bitmap bytes #54022

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zhannngchen wants to merge 1 commit into from
Closed

[opt](cloud-mow) add metrics for get/update delete bitmap bytes #54022

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion cloud/src/common/bvars.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

#include "common/bvars.h"
#include "bvars.h"

#include <bvar/multi_dimension.h>
#include <bvar/reducer.h>
Expand All @@ -25,6 +25,8 @@
#include <cstdint>
#include <stdexcept>

#include "common/bvars.h"

// clang-format off

// meta-service's bvars
Expand Down Expand Up @@ -91,6 +93,8 @@ BvarLatencyRecorderWithTag g_bvar_ms_get_cluster_status("ms", "get_cluster_statu
BvarLatencyRecorderWithTag g_bvar_ms_set_cluster_status("ms", "set_cluster_status");
BvarLatencyRecorderWithTag g_bvar_ms_check_kv("ms", "check_kv");
BvarLatencyRecorderWithTag g_bvar_ms_get_schema_dict("ms", "get_schema_dict");
BvarAdderWithTag<int64_t> g_bvar_ms_get_delete_bitmap_bytes("ms", "get_delete_bitmap_bytes");
BvarAdderWithTag<int64_t> g_bvar_ms_update_delete_bitmap_bytes("ms", "update_delete_bitmap_bytes");
bvar::Adder<int64_t> g_bvar_update_delete_bitmap_fail_counter;
bvar::Window<bvar::Adder<int64_t> > g_bvar_update_delete_bitmap_fail_counter_minute("ms", "update_delete_bitmap_fail", &g_bvar_update_delete_bitmap_fail_counter, 60);
bvar::Adder<int64_t> g_bvar_get_delete_bitmap_fail_counter;
Expand Down
18 changes: 15 additions & 3 deletions cloud/src/common/bvars.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,10 @@
* ${module}_${name}_${tag}
* where `tag` is added automatically when calling `get` or `put`
*/
template <typename Bvar, bool is_status = false>
template <typename Bvar, bool is_status = false, bool is_adder = false>
class BvarWithTag {
static_assert(!(is_status && is_adder), "is_status and is_adder cannot both be true");

public:
BvarWithTag(std::string module, std::string name)
: module_(std::move(module)), name_(std::move(name)) {}
Expand All @@ -52,14 +54,18 @@ class BvarWithTag {
std::lock_guard<bthread::Mutex> l(mutex_);
auto it = bvar_map_.find(tag);
if (it == bvar_map_.end()) {
instance = std::make_shared<Bvar>(module_, name_ + "_" + tag, ValType());
if constexpr (is_adder) {
instance = std::make_shared<Bvar>(module_, name_ + "_" + tag);
} else {
instance = std::make_shared<Bvar>(module_, name_ + "_" + tag, ValType());
}
bvar_map_[tag] = instance;
} else {
instance = it->second;
}
}
// FIXME(gavin): check bvar::Adder and more
if constexpr (std::is_same_v<Bvar, bvar::LatencyRecorder>) {
if constexpr (std::is_same_v<Bvar, bvar::LatencyRecorder> || is_adder) {
(*instance) << value;
} else if constexpr (is_status) {
instance->set_value(value);
Expand Down Expand Up @@ -98,6 +104,10 @@ class BvarWithTag {

using BvarLatencyRecorderWithTag = BvarWithTag<bvar::LatencyRecorder>;

template <typename T>
requires std::is_integral_v<T>
using BvarAdderWithTag = BvarWithTag<bvar::Adder<T>>;

template <typename T>
requires std::is_integral_v<T>
using BvarStatusWithTag = BvarWithTag<bvar::Status<T>, true>;
Expand Down Expand Up @@ -255,6 +265,8 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_check_kv;
extern BvarLatencyRecorderWithTag g_bvar_ms_get_schema_dict;
extern bvar::Adder<int64_t> g_bvar_update_delete_bitmap_fail_counter;
extern bvar::Adder<int64_t> g_bvar_get_delete_bitmap_fail_counter;
extern BvarAdderWithTag<int64_t> g_bvar_ms_get_delete_bitmap_bytes;
extern BvarAdderWithTag<int64_t> g_bvar_ms_update_delete_bitmap_bytes;

// recycler's bvars
extern BvarStatusWithTag<int64_t> g_bvar_recycler_recycle_index_earlest_ts;
Expand Down
10 changes: 10 additions & 0 deletions cloud/src/meta-service/meta_service.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
#include <unordered_map>
#include <utility>

#include "../common/bvars.h"
#include "common/bvars.h"
#include "common/config.h"
#include "common/encryption_util.h"
Expand Down Expand Up @@ -2375,6 +2376,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
<< " txn_size=" << txn->approximate_bytes();
msg = ss.str();
g_bvar_update_delete_bitmap_fail_counter << 1;
g_bvar_ms_update_delete_bitmap_bytes.put(instance_id, total_txn_put_bytes);
return;
}
stats.get_bytes += txn->get_bytes();
Expand All @@ -2389,6 +2391,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::CREATE>(err);
msg = "failed to init txn";
g_bvar_ms_update_delete_bitmap_bytes.put(instance_id, total_txn_put_bytes);
return;
}
if (!without_lock) {
Expand Down Expand Up @@ -2515,6 +2518,9 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
total_txn_put_bytes += txn->put_bytes();
total_txn_size += txn->approximate_bytes();
total_txn_count++;

g_bvar_ms_update_delete_bitmap_bytes.put(instance_id, total_txn_put_bytes);

if (err != TxnErrorCode::TXN_OK) {
if (err == TxnErrorCode::TXN_CONFLICT) {
g_bvar_delete_bitmap_lock_txn_put_conflict_counter << 1;
Expand Down Expand Up @@ -2629,6 +2635,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
code = cast_as<ErrCategory::CREATE>(err);
ss << "failed to init txn, retry=" << retry << ", internal round=" << round;
msg = ss.str();
g_bvar_ms_get_delete_bitmap_bytes.put(instance_id, delete_bitmap_byte);
return;
}
if (test) {
Expand All @@ -2648,6 +2655,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
<< ", ret=" << err;
msg = ss.str();
g_bvar_get_delete_bitmap_fail_counter << 1;
g_bvar_ms_get_delete_bitmap_bytes.put(instance_id, delete_bitmap_byte);
return;
}

Expand Down Expand Up @@ -2692,6 +2700,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
msg = ss.str();
LOG(WARNING) << msg;
g_bvar_get_delete_bitmap_fail_counter << 1;
g_bvar_ms_get_delete_bitmap_bytes.put(instance_id, delete_bitmap_byte);
return;
}
round++;
Expand All @@ -2705,6 +2714,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
LOG(INFO) << "finish get delete bitmap for tablet=" << tablet_id
<< ", delete_bitmap_num=" << delete_bitmap_num
<< ", delete_bitmap_byte=" << delete_bitmap_byte;
g_bvar_ms_get_delete_bitmap_bytes.put(instance_id, delete_bitmap_byte);

if (request->has_idx()) {
std::unique_ptr<Transaction> txn;
Expand Down