[fix](segment cache) estimate momory consumed by segment (#35647)

be/src/olap/base_tablet.cpp

@@ -33,6 +33,8 @@ extern MetricPrototype METRIC_query_scan_count;
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(flush_bytes, MetricUnit::BYTES);
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(flush_finish_count, MetricUnit::OPERATIONS);
 
+static bvar::Adder<size_t> g_total_tablet_num("doris_total_tablet_num");
+
 BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta) : _tablet_meta(std::move(tablet_meta)) {
     _metric_entity = DorisMetrics::instance()->metric_registry()->register_entity(
             fmt::format("Tablet.{}", tablet_id()), {{"tablet_id", std::to_string(tablet_id())}},
@@ -42,10 +44,12 @@ BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta) : _tablet_meta(std::move
     INT_COUNTER_METRIC_REGISTER(_metric_entity, query_scan_count);
     INT_COUNTER_METRIC_REGISTER(_metric_entity, flush_bytes);
     INT_COUNTER_METRIC_REGISTER(_metric_entity, flush_finish_count);
+    g_total_tablet_num << 1;
 }
 
 BaseTablet::~BaseTablet() {
     DorisMetrics::instance()->metric_registry()->deregister_entity(_metric_entity);
+    g_total_tablet_num << -1;
 }
 
 Status BaseTablet::set_tablet_state(TabletState state) {

be/src/olap/primary_key_index.cpp

@@ -32,6 +32,8 @@
 
 namespace doris {
 
+static bvar::Adder<size_t> g_primary_key_index_memory_bytes("doris_primary_key_index_memory_bytes");
+
 Status PrimaryKeyIndexBuilder::init() {
     // TODO(liaoxin) using the column type directly if there's only one column in unique key columns
     const auto* type_info = get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARCHAR>();

be/src/olap/rowset/rowset.cpp

@@ -26,6 +26,8 @@
 
 namespace doris {
 
+static bvar::Adder<size_t> g_total_rowset_num("doris_total_rowset_num");
+
 Rowset::Rowset(const TabletSchemaSPtr& schema, const RowsetMetaSharedPtr& rowset_meta)
         : _rowset_meta(rowset_meta), _refs_by_reader(0) {
     _is_pending = !_rowset_meta->has_version();
@@ -37,6 +39,11 @@ Rowset::Rowset(const TabletSchemaSPtr& schema, const RowsetMetaSharedPtr& rowset
     }
     // build schema from RowsetMeta.tablet_schema or Tablet.tablet_schema
     _schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() : schema;
+    g_total_rowset_num << 1;
+}
+
+Rowset::~Rowset() {
+    g_total_rowset_num << -1;
 }
 
 Status Rowset::load(bool use_cache) {

be/src/olap/rowset/rowset.h

@@ -118,7 +118,7 @@ class RowsetStateMachine {
 
 class Rowset : public std::enable_shared_from_this<Rowset> {
 public:
-    virtual ~Rowset() = default;
+    virtual ~Rowset();
 
     // Open all segment files in this rowset and load necessary metadata.
     // - `use_cache` : whether to use fd cache, only applicable to alpha rowset now

be/src/olap/rowset/segment_v2/column_reader.cpp

@@ -76,6 +76,9 @@
 namespace doris {
 namespace segment_v2 {
 
+static bvar::Adder<size_t> g_column_reader_memory_bytes("doris_column_reader_memory_bytes");
+static bvar::Adder<size_t> g_column_reader_num("doris_column_reader_num");
+
 Status ColumnReader::create(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
                             uint64_t num_rows, const io::FileReaderSPtr& file_reader,
                             std::unique_ptr<ColumnReader>* reader) {
@@ -205,9 +208,15 @@ ColumnReader::ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB&
     _meta_is_nullable = meta.is_nullable();
     _meta_dict_page = meta.dict_page();
     _meta_compression = meta.compression();
+
+    g_column_reader_memory_bytes << sizeof(*this);
+    g_column_reader_num << 1;
 }
 
-ColumnReader::~ColumnReader() = default;
+ColumnReader::~ColumnReader() {
+    g_column_reader_memory_bytes << -sizeof(*this);
+    g_column_reader_num << -1;
+}
 
 Status ColumnReader::init(const ColumnMetaPB* meta) {
     _type_info = get_type_info(meta);

be/src/olap/rowset/segment_v2/column_reader.h

@@ -116,7 +116,7 @@ class ColumnReader {
 
     enum DictEncodingType { UNKNOWN_DICT_ENCODING, PARTIAL_DICT_ENCODING, ALL_DICT_ENCODING };
 
-    ~ColumnReader();
+    virtual ~ColumnReader();
 
     // create a new column iterator. Client should delete returned iterator
     Status new_iterator(ColumnIterator** iterator);

be/src/olap/rowset/segment_v2/indexed_column_reader.cpp

@@ -56,6 +56,8 @@ static bvar::Adder<uint64_t> g_index_reader_pk_pages("doris_pk", "index_reader_p
 static bvar::PerSecond<bvar::Adder<uint64_t>> g_index_reader_pk_bytes_per_second(
         "doris_pk", "index_reader_pk_pages_per_second", &g_index_reader_pk_pages, 60);
 
+static bvar::Adder<uint64_t> g_index_reader_memory_bytes("doris_index_reader_memory_bytes");
+
 using strings::Substitute;
 
 Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory) {
@@ -91,6 +93,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory) {
         }
     }
     _num_values = _meta.num_values();
+
+    g_index_reader_memory_bytes << sizeof(*this);
     return Status::OK();
 }
 
@@ -134,6 +138,10 @@ Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle,
     return st;
 }
 
+IndexedColumnReader::~IndexedColumnReader() {
+    g_index_reader_memory_bytes << -sizeof(*this);
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 
 Status IndexedColumnIterator::_read_data_page(const PagePointer& pp) {

be/src/olap/rowset/segment_v2/indexed_column_reader.h

@@ -51,6 +51,8 @@ class IndexedColumnReader {
     explicit IndexedColumnReader(io::FileReaderSPtr file_reader, const IndexedColumnMetaPB& meta)
             : _file_reader(std::move(file_reader)), _meta(meta) {}
 
+    ~IndexedColumnReader();
+
     Status load(bool use_page_cache, bool kept_in_memory);
 
     // read a page specified by `pp' from `file' into `handle'

be/src/olap/rowset/segment_v2/ordinal_page_index.cpp

@@ -29,9 +29,13 @@
 #include "olap/olap_common.h"
 #include "olap/rowset/segment_v2/page_handle.h"
 #include "olap/rowset/segment_v2/page_io.h"
+#include "ordinal_page_index.h"
 #include "util/slice.h"
 
 namespace doris {
+
+static bvar::Adder<size_t> g_ordinal_index_memory_bytes("doris_ordinal_index_memory_bytes");
+
 namespace segment_v2 {
 
 void OrdinalIndexWriter::append_entry(ordinal_t ordinal, const PagePointer& data_pp) {
@@ -122,6 +126,10 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool kept_in_memory,
         _pages[i] = reader.get_value(i);
     }
     _ordinals[_num_pages] = _num_values;
+
+    g_ordinal_index_memory_bytes << sizeof(*this) + _ordinals.size() * sizeof(ordinal_t) +
+                                            _pages.size() * sizeof(PagePointer) +
+                                            sizeof(OrdinalIndexReader);
     return Status::OK();
 }
 
@@ -146,5 +154,11 @@ OrdinalPageIndexIterator OrdinalIndexReader::seek_at_or_before(ordinal_t ordinal
     return OrdinalPageIndexIterator(this, left);
 }
 
+OrdinalIndexReader::~OrdinalIndexReader() {
+    g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() * sizeof(ordinal_t) -
+                                            _pages.size() * sizeof(PagePointer) -
+                                            sizeof(OrdinalIndexReader);
+}
+
 } // namespace segment_v2
 } // namespace doris

be/src/olap/rowset/segment_v2/ordinal_page_index.h

@@ -72,6 +72,8 @@ class OrdinalIndexReader {
         _meta_pb.reset(new OrdinalIndexPB(meta_pb));
     }
 
+    virtual ~OrdinalIndexReader();
+
     // load and parse the index page into memory
     Status load(bool use_page_cache, bool kept_in_memory);
 

be/src/olap/rowset/segment_v2/segment.cpp

@@ -113,6 +113,17 @@ Status Segment::_open() {
     // DCHECK(footer.has_short_key_index_page());
     _sk_index_page = _footer_pb->short_key_index_page();
     _num_rows = _footer_pb->num_rows();
+
+    // An estimated memory usage of a segment
+    _meta_mem_usage += _footer_pb->ByteSizeLong();
+    _meta_mem_usage += sizeof(*this);
+    _meta_mem_usage += _tablet_schema->num_columns() * config::estimated_mem_per_column_reader;
+
+    // 1024 comes from SegmentWriterOptions
+    _meta_mem_usage += (_num_rows + 1023) / 1024 * (36 + 4);
+    // 0.01 comes from PrimaryKeyIndexBuilder::init
+    _meta_mem_usage += BloomFilter::optimal_bit_num(_num_rows, 0.01) / 8;
+
     return Status::OK();
 }
 
@@ -301,7 +312,7 @@ Status Segment::_load_pk_bloom_filter() {
     auto status = [this]() {
         return _load_pk_bf_once.call([this] {
             RETURN_IF_ERROR(_pk_index_reader->parse_bf(_file_reader, *_pk_index_meta));
-            _meta_mem_usage += _pk_index_reader->get_bf_memory_size();
+            // _meta_mem_usage += _pk_index_reader->get_bf_memory_size();
             return Status::OK();
         });
     }();
@@ -338,7 +349,7 @@ Status Segment::_load_index_impl() {
         if (_tablet_schema->keys_type() == UNIQUE_KEYS && _pk_index_meta != nullptr) {
             _pk_index_reader.reset(new PrimaryKeyIndexReader());
             RETURN_IF_ERROR(_pk_index_reader->parse_index(_file_reader, *_pk_index_meta));
-            _meta_mem_usage += _pk_index_reader->get_memory_size();
+            // _meta_mem_usage += _pk_index_reader->get_memory_size();
             return Status::OK();
         } else {
             // read and parse short key index page
@@ -360,7 +371,7 @@ Status Segment::_load_index_impl() {
             DCHECK_EQ(footer.type(), SHORT_KEY_PAGE);
             DCHECK(footer.has_short_key_page_footer());
 
-            _meta_mem_usage += body.get_size();
+            // _meta_mem_usage += body.get_size();
             _sk_index_decoder.reset(new ShortKeyIndexDecoder);
             return _sk_index_decoder->parse(body, footer.short_key_page_footer());
         }
@@ -430,7 +441,6 @@ Status Segment::_create_column_readers(const SegmentFooterPB& footer) {
         RETURN_IF_ERROR(ColumnReader::create(opts, footer.columns(iter->second), footer.num_rows(),
                                              _file_reader, &reader));
         _column_readers.emplace(column.unique_id(), std::move(reader));
-        _meta_mem_usage += config::estimated_mem_per_column_reader;
     }
 
     // init by column path

be/src/olap/rowset/segment_v2/segment.h

@@ -256,6 +256,7 @@ class Segment : public std::enable_shared_from_this<Segment> {
     // used to hold short key index page in memory
     PageHandle _sk_index_handle;
     // short key index decoder
+    // all content is in memory
     std::unique_ptr<ShortKeyIndexDecoder> _sk_index_decoder;
     // primary key index reader
     std::unique_ptr<PrimaryKeyIndexReader> _pk_index_reader;

be/src/olap/rowset/segment_v2/zone_map_index.cpp

@@ -39,6 +39,8 @@
 namespace doris {
 struct uint24_t;
 
+static bvar::Adder<size_t> g_zone_map_memory_bytes("doris_zone_map_memory_bytes");
+
 namespace segment_v2 {
 
 template <PrimitiveType Type>
@@ -173,9 +175,18 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory,
             return Status::Corruption("Failed to parse zone map");
         }
     }
+
+    g_zone_map_memory_bytes << sizeof(*this) + sizeof(ZoneMapPB) * _page_zone_maps.size() +
+                                       sizeof(IndexedColumnMetaPB);
+
     return Status::OK();
 }
 
+ZoneMapIndexReader::~ZoneMapIndexReader() {
+    // Maybe wrong due to load failures.
+    g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) * _page_zone_maps.size() -
+                                       sizeof(IndexedColumnMetaPB);
+}
 #define APPLY_FOR_PRIMITITYPE(M) \
     M(TYPE_TINYINT)              \
     M(TYPE_SMALLINT)             \

be/src/olap/rowset/segment_v2/zone_map_index.h

@@ -151,6 +151,8 @@ class ZoneMapIndexReader {
         _page_zone_maps_meta.reset(new IndexedColumnMetaPB(page_zone_maps));
     }
 
+    virtual ~ZoneMapIndexReader();
+
     // load all page zone maps into memory
     Status load(bool use_page_cache, bool kept_in_memory);
 

be/src/olap/short_key_index.cpp

@@ -22,12 +22,16 @@
 #include <ostream>
 
 #include "gutil/strings/substitute.h"
+#include "short_key_index.h"
+#include "util/bvar_helper.h"
 #include "util/coding.h"
 
 using strings::Substitute;
 
 namespace doris {
 
+static bvar::Adder<size_t> g_short_key_index_memory_bytes("doris_short_key_index_memory_bytes");
+
 Status ShortKeyIndexBuilder::add_item(const Slice& key) {
     put_varint32(&_offset_buf, _key_buf.size());
     _key_buf.append(key.data, key.size);
@@ -85,7 +89,19 @@ Status ShortKeyIndexDecoder::parse(const Slice& body, const segment_v2::ShortKey
         return Status::Corruption("Still has data after parse all key offset");
     }
     _parsed = true;
+
+    g_short_key_index_memory_bytes << sizeof(_footer) + _key_data.size +
+                                              _offsets.size() * sizeof(uint32_t) + sizeof(*this);
+
     return Status::OK();
 }
 
+ShortKeyIndexDecoder::~ShortKeyIndexDecoder() {
+    if (_parsed) {
+        g_short_key_index_memory_bytes << -sizeof(_footer) - _key_data.size -
+                                                  _offsets.size() * sizeof(uint32_t) -
+                                                  sizeof(*this);
+    }
+}
+
 } // namespace doris

be/src/olap/short_key_index.h

@@ -135,6 +135,7 @@ class ShortKeyIndexIterator {
 class ShortKeyIndexDecoder {
 public:
     ShortKeyIndexDecoder() : _parsed(false) {}
+    virtual ~ShortKeyIndexDecoder();
 
     // client should assure that body is available when this class is used
     Status parse(const Slice& body, const segment_v2::ShortKeyFooterPB& footer);

be/src/olap/tablet_schema.cpp

@@ -53,6 +53,8 @@
 
 namespace doris {
 
+static bvar::Adder<size_t> g_total_tablet_schema_num("doris_total_tablet_schema_num");
+
 FieldType TabletColumn::get_field_type_by_type(PrimitiveType primitiveType) {
     switch (primitiveType) {
     case PrimitiveType::INVALID_TYPE:
@@ -809,6 +811,14 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const {
     }
 }
 
+TabletSchema::TabletSchema() {
+    g_total_tablet_schema_num << 1;
+}
+
+TabletSchema::~TabletSchema() {
+    g_total_tablet_schema_num << -1;
+}
+
 void TabletSchema::append_column(TabletColumn column, ColumnType col_type) {
     if (column.is_key()) {
         _num_key_columns++;

be/src/olap/tablet_schema.h

@@ -269,7 +269,9 @@ class TabletSchema {
     // TODO(yingchun): better to make constructor as private to avoid
     // manually init members incorrectly, and define a new function like
     // void create_from_pb(const TabletSchemaPB& schema, TabletSchema* tablet_schema).
-    TabletSchema() = default;
+    TabletSchema();
+    virtual ~TabletSchema();
+
     void init_from_pb(const TabletSchemaPB& schema, bool ignore_extracted_columns = false);
     // Notice: Use deterministic way to serialize protobuf,
     // since serialize Map in protobuf may could lead to un-deterministic by default