From 1cd551ea0007083c3e8acb825ffcd20feb2c8651 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Tue, 20 Aug 2024 16:03:49 +0800 Subject: [PATCH 1/3] [fix](stat) handle overflow of memory stat if load failed --- be/src/olap/rowset/segment_v2/ordinal_page_index.cpp | 12 ++++++++---- be/src/olap/rowset/segment_v2/zone_map_index.cpp | 10 ++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp index b93d461d8c78a0..777e74d5825d16 100644 --- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp +++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp @@ -115,6 +115,11 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool kept_in_memory, _num_pages = reader.count(); _ordinals.resize(_num_pages + 1); _pages.resize(_num_pages); + + g_ordinal_index_memory_bytes << sizeof(*this) + _ordinals.size() * sizeof(ordinal_t) + + _pages.size() * sizeof(PagePointer) + + sizeof(OrdinalIndexReader); + for (int i = 0; i < _num_pages; i++) { Slice key = reader.get_key(i); ordinal_t ordinal = 0; @@ -127,9 +132,6 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool kept_in_memory, } _ordinals[_num_pages] = _num_values; - g_ordinal_index_memory_bytes << sizeof(*this) + _ordinals.size() * sizeof(ordinal_t) + - _pages.size() * sizeof(PagePointer) + - sizeof(OrdinalIndexReader); return Status::OK(); } @@ -155,9 +157,11 @@ OrdinalPageIndexIterator OrdinalIndexReader::seek_at_or_before(ordinal_t ordinal } OrdinalIndexReader::~OrdinalIndexReader() { - g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() * sizeof(ordinal_t) - + if (_ordinals.size() > 0) { + g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() * sizeof(ordinal_t) - _pages.size() * sizeof(PagePointer) - sizeof(OrdinalIndexReader); + } } } // namespace segment_v2 diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp b/be/src/olap/rowset/segment_v2/zone_map_index.cpp index 6a1dee39cd76b6..e9e4eed809b8c2 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp +++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp @@ -157,6 +157,9 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory, _page_zone_maps.resize(reader.num_values()); + g_zone_map_memory_bytes << sizeof(*this) + sizeof(ZoneMapPB) * _page_zone_maps.size() + + sizeof(IndexedColumnMetaPB); + // read and cache all page zone maps for (int i = 0; i < reader.num_values(); ++i) { size_t num_to_read = 1; @@ -176,16 +179,15 @@ Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory, } } - g_zone_map_memory_bytes << sizeof(*this) + sizeof(ZoneMapPB) * _page_zone_maps.size() + - sizeof(IndexedColumnMetaPB); - return Status::OK(); } ZoneMapIndexReader::~ZoneMapIndexReader() { // Maybe wrong due to load failures. - g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) * _page_zone_maps.size() - + if (_page_zone_maps.size() > 0) { + g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) * _page_zone_maps.size() - sizeof(IndexedColumnMetaPB); + } } #define APPLY_FOR_PRIMITITYPE(M) \ M(TYPE_TINYINT) \ From e4416bb9f665982c26477427bc2ef204bf6a5576 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Tue, 20 Aug 2024 23:47:11 +0800 Subject: [PATCH 2/3] fix --- be/src/olap/rowset/segment_v2/ordinal_page_index.cpp | 4 ++-- be/src/olap/rowset/segment_v2/zone_map_index.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp index 777e74d5825d16..24b2e3379963bc 100644 --- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp +++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp @@ -159,8 +159,8 @@ OrdinalPageIndexIterator OrdinalIndexReader::seek_at_or_before(ordinal_t ordinal OrdinalIndexReader::~OrdinalIndexReader() { if (_ordinals.size() > 0) { g_ordinal_index_memory_bytes << -sizeof(*this) - _ordinals.size() * sizeof(ordinal_t) - - _pages.size() * sizeof(PagePointer) - - sizeof(OrdinalIndexReader); + _pages.size() * sizeof(PagePointer) - + sizeof(OrdinalIndexReader); } } diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp b/be/src/olap/rowset/segment_v2/zone_map_index.cpp index e9e4eed809b8c2..e625178288c045 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp +++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp @@ -186,7 +186,7 @@ ZoneMapIndexReader::~ZoneMapIndexReader() { // Maybe wrong due to load failures. if (_page_zone_maps.size() > 0) { g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) * _page_zone_maps.size() - - sizeof(IndexedColumnMetaPB); + sizeof(IndexedColumnMetaPB); } } #define APPLY_FOR_PRIMITITYPE(M) \ From 58bfef07cd55fcca42ad738aa729e024d4e07147 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Wed, 21 Aug 2024 09:30:37 +0800 Subject: [PATCH 3/3] fix --- be/src/olap/rowset/segment_v2/zone_map_index.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp b/be/src/olap/rowset/segment_v2/zone_map_index.cpp index e625178288c045..991df2f94755c9 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp +++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp @@ -186,7 +186,7 @@ ZoneMapIndexReader::~ZoneMapIndexReader() { // Maybe wrong due to load failures. if (_page_zone_maps.size() > 0) { g_zone_map_memory_bytes << -sizeof(*this) - sizeof(ZoneMapPB) * _page_zone_maps.size() - - sizeof(IndexedColumnMetaPB); + sizeof(IndexedColumnMetaPB); } } #define APPLY_FOR_PRIMITITYPE(M) \