From da88c23f11360b257e8da85d326b999fc2f21378 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Fri, 16 Aug 2024 09:58:50 +0800 Subject: [PATCH 1/4] [Fix](tablet-meta)limiting the size of tablet meta --- be/src/common/config.cpp | 7 +++++++ be/src/common/config.h | 2 ++ be/src/olap/push_handler.cpp | 11 +++++++++++ be/src/olap/rowset_builder.cpp | 11 +++++++++++ be/src/olap/schema_change.cpp | 2 +- be/src/olap/tablet.h | 5 +++++ be/src/olap/tablet_meta.cpp | 4 ++++ be/src/olap/tablet_meta.h | 4 ++++ 8 files changed, 45 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 150ec224f31778..752890b9032b7b 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1343,6 +1343,13 @@ DEFINE_mBool(enable_hdfs_mem_limiter, "true"); DEFINE_mInt16(topn_agg_limit_multiplier, "2"); +// Tablet meta size limit after serialization, 1.5GB +DEFINE_mInt64(tablet_meta_serialize_size_limit, "1610612736"); +// Protobuf supports a maximum of 2GB, so the size of the tablet meta after serialization must be less than 2GB +// 1717986918 = 2GB * 0.8 +DEFINE_Validator(tablet_meta_serialize_size_limit, + [](const int64_t config) -> bool { return config < 1717986918; }); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 135daffcea5927..3710cb1d1010cf 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1438,6 +1438,8 @@ DECLARE_mBool(enable_hdfs_mem_limiter); // we should do agg limit opt DECLARE_mInt16(topn_agg_limit_multiplier); +DECLARE_mInt64(tablet_meta_serialize_size_limit); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index feb7d24dda2705..65d336c8e7fc34 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -166,6 +166,17 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR "failed to push data. version count: {}, exceed limit: {}, tablet: {}", tablet->version_count(), config::max_tablet_version_num, tablet->tablet_id()); } + + int version_count = tablet->version_count(); + if (tablet->avg_rs_meta_serialize_size() * version_count > + config::tablet_meta_serialize_size_limit) { + return Status::Error( + "failed to init rowset builder. meta serialize size : {}, exceed limit: {}, " + "tablet: {}", + tablet->avg_rs_meta_serialize_size() * version_count, + config::tablet_meta_serialize_size_limit, tablet->tablet_id()); + } + auto tablet_schema = std::make_shared(); tablet_schema->copy_from(*tablet->tablet_schema()); if (!request.columns_desc.empty() && request.columns_desc[0].col_unique_id >= 0) { diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index f80c6457215b52..e1d6c763d62bf8 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -168,6 +168,7 @@ Status RowsetBuilder::check_tablet_version_count() { "tablet: {}", version_count, config::max_tablet_version_num, _tablet->tablet_id()); } + return Status::OK(); } @@ -195,6 +196,16 @@ Status RowsetBuilder::init() { RETURN_IF_ERROR(check_tablet_version_count()); } + int version_count = tablet()->version_count(); + if (tablet()->avg_rs_meta_serialize_size() * version_count > + config::tablet_meta_serialize_size_limit) { + return Status::Error( + "failed to init rowset builder. meta serialize size : {}, exceed limit: {}, " + "tablet: {}", + tablet()->avg_rs_meta_serialize_size() * version_count, + config::tablet_meta_serialize_size_limit, _tablet->tablet_id()); + } + RETURN_IF_ERROR(prepare_txn()); DBUG_EXECUTE_IF("BaseRowsetBuilder::init.check_partial_update_column_num", { diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 1771bfb7c6714c..04409e37d84c3c 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -894,7 +894,7 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques } } std::vector empty_vec; - RETURN_IF_ERROR(_new_tablet->modify_rowsets(empty_vec, rowsets_to_delete)); + _new_tablet->delete_rowsets(rowsets_to_delete, false); // inherit cumulative_layer_point from base_tablet // check if new_tablet.ce_point > base_tablet.ce_point? _new_tablet->set_cumulative_layer_point(-1); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index fa11c2d868569f..fc85f3b4f5a689 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -157,6 +157,7 @@ class Tablet final : public BaseTablet { double bloom_filter_fpp() const; size_t next_unique_id() const; size_t row_size() const; + int64_t avg_rs_meta_serialize_size() const; // operation in rowsets Status add_rowset(RowsetSharedPtr rowset); @@ -723,4 +724,8 @@ inline size_t Tablet::row_size() const { return _tablet_meta->tablet_schema()->row_size(); } +inline int64_t Tablet::avg_rs_meta_serialize_size() const { + return _tablet_meta->avg_rs_meta_serialize_size(); +} + } // namespace doris diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index ed9a446551d00e..3b486076253fa0 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -535,6 +535,10 @@ void TabletMeta::serialize(string* meta_binary) { if (!serialize_success) { LOG(FATAL) << "failed to serialize meta " << tablet_id(); } + if (!_rs_metas.empty()) { + _avg_rs_meta_serialize_size = + meta_binary->length() / (_rs_metas.size() + _stale_rs_metas.size()); + } } Status TabletMeta::deserialize(std::string_view meta_binary) { diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 41455c051c7f44..e83b03ec3ca8b9 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -284,6 +284,8 @@ class TabletMeta { _ttl_seconds = ttl_seconds; } + int64_t avg_rs_meta_serialize_size() const { return _avg_rs_meta_serialize_size; } + private: Status _save_meta(DataDir* data_dir); @@ -339,6 +341,8 @@ class TabletMeta { int64_t _time_series_compaction_empty_rowsets_threshold = 0; int64_t _time_series_compaction_level_threshold = 0; + int64_t _avg_rs_meta_serialize_size = 0; + // cloud int64_t _ttl_seconds = 0; From 06fe49d5b5e00317c5484621230dc599b0492244 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Fri, 23 Aug 2024 13:10:00 +0800 Subject: [PATCH 2/4] fix --- be/src/olap/push_handler.cpp | 2 +- be/src/olap/rowset_builder.cpp | 3 +-- be/src/olap/tablet.h | 6 ++++++ be/src/olap/tablet_meta.cpp | 19 +++++++++++++++++++ be/src/olap/tablet_meta.h | 5 +++++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index 65d336c8e7fc34..aef701a0545ac9 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -167,7 +167,7 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR tablet->version_count(), config::max_tablet_version_num, tablet->tablet_id()); } - int version_count = tablet->version_count(); + int version_count = tablet->version_count() + tablet->stale_version_count(); if (tablet->avg_rs_meta_serialize_size() * version_count > config::tablet_meta_serialize_size_limit) { return Status::Error( diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index e1d6c763d62bf8..39fcc3f6c231ab 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -168,7 +168,6 @@ Status RowsetBuilder::check_tablet_version_count() { "tablet: {}", version_count, config::max_tablet_version_num, _tablet->tablet_id()); } - return Status::OK(); } @@ -196,7 +195,7 @@ Status RowsetBuilder::init() { RETURN_IF_ERROR(check_tablet_version_count()); } - int version_count = tablet()->version_count(); + int version_count = tablet()->version_count() + tablet()->stale_version_count(); if (tablet()->avg_rs_meta_serialize_size() * version_count > config::tablet_meta_serialize_size_limit) { return Status::Error( diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index fc85f3b4f5a689..ff20503b7bd2e9 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -141,6 +141,7 @@ class Tablet final : public BaseTablet { size_t num_rows(); int version_count() const; + int stale_version_count() const; bool exceed_version_limit(int32_t limit) override; uint64_t segment_count() const; Version max_version() const; @@ -670,6 +671,11 @@ inline int Tablet::version_count() const { return _tablet_meta->version_count(); } +inline int Tablet::stale_version_count() const { + std::shared_lock rdlock(_meta_lock); + return _tablet_meta->stale_version_count(); +} + inline Version Tablet::max_version() const { std::shared_lock rdlock(_meta_lock); return _tablet_meta->max_version(); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 3b486076253fa0..34838c4c225018 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -26,11 +26,13 @@ #include #include +#include #include #include #include "cloud/config.h" #include "common/config.h" +#include "gutil/integral_types.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "olap/data_dir.h" @@ -538,6 +540,23 @@ void TabletMeta::serialize(string* meta_binary) { if (!_rs_metas.empty()) { _avg_rs_meta_serialize_size = meta_binary->length() / (_rs_metas.size() + _stale_rs_metas.size()); + + if (meta_binary->length() > config::tablet_meta_serialize_size_limit) { + int64_t origin_meta_size = meta_binary->length(); + int64_t stale_rowsets_num = tablet_meta_pb.stale_rs_metas().size(); + tablet_meta_pb.clear_stale_rs_metas(); + meta_binary->clear(); + serialize_success = tablet_meta_pb.SerializeToString(meta_binary); + LOG(WARNING) << "tablet meta serialization size exceeds limit: " + << config::tablet_meta_serialize_size_limit + << " clean up stale rowsets, tablet id: " << tablet_id() + << " stale rowset num: " << stale_rowsets_num + << " serialization size before clean " << origin_meta_size + << " serialization size after clean " << meta_binary->length(); + if (!serialize_success) { + LOG(FATAL) << "failed to serialize meta " << tablet_id(); + } + } } } diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index e83b03ec3ca8b9..74ab71d0586fa0 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -165,6 +165,7 @@ class TabletMeta { // Remote disk space occupied by tablet. size_t tablet_remote_size() const; size_t version_count() const; + size_t stale_version_count() const; size_t version_count_cross_with_range(const Version& range) const; Version max_version() const; @@ -649,6 +650,10 @@ inline size_t TabletMeta::version_count() const { return _rs_metas.size(); } +inline size_t TabletMeta::stale_version_count() const { + return _rs_metas.size(); +} + inline TabletState TabletMeta::tablet_state() const { return _tablet_state; } From be27f7c9f89c198d25770da2195eed64962363ac Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Fri, 23 Aug 2024 21:57:49 +0800 Subject: [PATCH 3/4] fix --- be/src/olap/tablet_meta.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 34838c4c225018..775f337e15087f 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -534,14 +534,10 @@ void TabletMeta::serialize(string* meta_binary) { << partition_id << " new=" << tablet_meta_pb.DebugString(); }); bool serialize_success = tablet_meta_pb.SerializeToString(meta_binary); - if (!serialize_success) { - LOG(FATAL) << "failed to serialize meta " << tablet_id(); - } - if (!_rs_metas.empty()) { + if (!_rs_metas.empty() || !_stale_rs_metas.empty()) { _avg_rs_meta_serialize_size = meta_binary->length() / (_rs_metas.size() + _stale_rs_metas.size()); - - if (meta_binary->length() > config::tablet_meta_serialize_size_limit) { + if (meta_binary->length() > config::tablet_meta_serialize_size_limit || !serialize_success) { int64_t origin_meta_size = meta_binary->length(); int64_t stale_rowsets_num = tablet_meta_pb.stale_rs_metas().size(); tablet_meta_pb.clear_stale_rs_metas(); @@ -553,11 +549,12 @@ void TabletMeta::serialize(string* meta_binary) { << " stale rowset num: " << stale_rowsets_num << " serialization size before clean " << origin_meta_size << " serialization size after clean " << meta_binary->length(); - if (!serialize_success) { - LOG(FATAL) << "failed to serialize meta " << tablet_id(); - } } } + + if (!serialize_success) { + LOG(FATAL) << "failed to serialize meta " << tablet_id(); + } } Status TabletMeta::deserialize(std::string_view meta_binary) { From b380de3b76021ba11a9aa569558b8983c445ceb1 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Fri, 23 Aug 2024 21:58:37 +0800 Subject: [PATCH 4/4] fix --- be/src/olap/tablet_meta.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 775f337e15087f..19302da7117aaa 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -537,7 +537,8 @@ void TabletMeta::serialize(string* meta_binary) { if (!_rs_metas.empty() || !_stale_rs_metas.empty()) { _avg_rs_meta_serialize_size = meta_binary->length() / (_rs_metas.size() + _stale_rs_metas.size()); - if (meta_binary->length() > config::tablet_meta_serialize_size_limit || !serialize_success) { + if (meta_binary->length() > config::tablet_meta_serialize_size_limit || + !serialize_success) { int64_t origin_meta_size = meta_binary->length(); int64_t stale_rowsets_num = tablet_meta_pb.stale_rs_metas().size(); tablet_meta_pb.clear_stale_rs_metas();