From 2085057a4d994fc5c4cd9c2538f8fb20838ce212 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 20 Dec 2024 20:27:18 +0800 Subject: [PATCH 1/2] [opt](profile) add index page profile for io (#45675) --- be/src/olap/base_tablet.cpp | 4 +- be/src/olap/delete_bitmap_calculator.cpp | 5 +- be/src/olap/primary_key_index.cpp | 12 ++-- be/src/olap/primary_key_index.h | 12 ++-- .../segment_v2/bloom_filter_index_reader.cpp | 15 ++--- .../segment_v2/bloom_filter_index_reader.h | 12 ++-- .../olap/rowset/segment_v2/column_reader.cpp | 62 +++++++++++-------- be/src/olap/rowset/segment_v2/column_reader.h | 26 +++++--- .../segment_v2/indexed_column_reader.cpp | 12 ++-- .../rowset/segment_v2/indexed_column_reader.h | 4 +- .../rowset/segment_v2/ordinal_page_index.cpp | 16 +++-- .../rowset/segment_v2/ordinal_page_index.h | 5 +- be/src/olap/rowset/segment_v2/segment.cpp | 35 ++++++----- be/src/olap/rowset/segment_v2/segment.h | 9 ++- .../rowset/segment_v2/segment_iterator.cpp | 2 +- .../olap/rowset/segment_v2/zone_map_index.cpp | 15 +++-- .../olap/rowset/segment_v2/zone_map_index.h | 6 +- be/test/olap/date_bloom_filter_test.cpp | 8 +-- be/test/olap/primary_key_index_test.cpp | 20 +++--- .../bloom_filter_index_reader_writer_test.cpp | 4 +- .../segment_v2/ordinal_page_index_test.cpp | 4 +- be/test/olap/segment_cache_test.cpp | 2 +- 22 files changed, 162 insertions(+), 128 deletions(-) diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index d4360f7da96f54..a5df20d63835df 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -593,7 +593,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, vectorized::Block ordered_block = block.clone_empty(); uint32_t pos = 0; - RETURN_IF_ERROR(seg->load_pk_index_and_bf()); // We need index blocks to iterate + RETURN_IF_ERROR(seg->load_pk_index_and_bf(nullptr)); // We need index blocks to iterate const auto* pk_idx = seg->get_primary_key_index(); int total = pk_idx->num_rows(); uint32_t row_id = 0; @@ -607,7 +607,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, std::vector> segment_caches(specified_rowsets.size()); while (remaining > 0) { std::unique_ptr iter; - RETURN_IF_ERROR(pk_idx->new_iterator(&iter)); + RETURN_IF_ERROR(pk_idx->new_iterator(&iter, nullptr)); size_t num_to_read = std::min(batch_size, remaining); auto index_type = vectorized::DataTypeFactory::instance().create_data_type( diff --git a/be/src/olap/delete_bitmap_calculator.cpp b/be/src/olap/delete_bitmap_calculator.cpp index 017e3cff3d0489..8ac05a1e393043 100644 --- a/be/src/olap/delete_bitmap_calculator.cpp +++ b/be/src/olap/delete_bitmap_calculator.cpp @@ -145,12 +145,11 @@ Status MergeIndexDeleteBitmapCalculator::init(RowsetId rowset_id, MergeIndexDeleteBitmapCalculatorContext::Comparator(seq_col_length, _rowid_length); _contexts.reserve(segments.size()); _heap = std::make_unique(_comparator); - for (auto& segment : segments) { - RETURN_IF_ERROR(segment->load_index()); + RETURN_IF_ERROR(segment->load_index(nullptr)); auto pk_idx = segment->get_primary_key_index(); std::unique_ptr index; - RETURN_IF_ERROR(pk_idx->new_iterator(&index)); + RETURN_IF_ERROR(pk_idx->new_iterator(&index, nullptr)); auto index_type = vectorized::DataTypeFactory::instance().create_data_type( pk_idx->type_info()->type(), 1, 0); _contexts.emplace_back(std::move(index), index_type, segment->id(), pk_idx->num_rows()); diff --git a/be/src/olap/primary_key_index.cpp b/be/src/olap/primary_key_index.cpp index 9d40ff5a8fad51..f2cd08fee8a910 100644 --- a/be/src/olap/primary_key_index.cpp +++ b/be/src/olap/primary_key_index.cpp @@ -92,27 +92,29 @@ Status PrimaryKeyIndexBuilder::finalize(segment_v2::PrimaryKeyIndexMetaPB* meta) } Status PrimaryKeyIndexReader::parse_index(io::FileReaderSPtr file_reader, - const segment_v2::PrimaryKeyIndexMetaPB& meta) { + const segment_v2::PrimaryKeyIndexMetaPB& meta, + OlapReaderStatistics* pk_index_load_stats) { // parse primary key index _index_reader.reset(new segment_v2::IndexedColumnReader(file_reader, meta.primary_key_index())); _index_reader->set_is_pk_index(true); RETURN_IF_ERROR(_index_reader->load(!config::disable_pk_storage_page_cache, false, - _pk_index_load_stats)); + pk_index_load_stats)); _index_parsed = true; return Status::OK(); } Status PrimaryKeyIndexReader::parse_bf(io::FileReaderSPtr file_reader, - const segment_v2::PrimaryKeyIndexMetaPB& meta) { + const segment_v2::PrimaryKeyIndexMetaPB& meta, + OlapReaderStatistics* pk_index_load_stats) { // parse bloom filter segment_v2::ColumnIndexMetaPB column_index_meta = meta.bloom_filter_index(); segment_v2::BloomFilterIndexReader bf_index_reader(std::move(file_reader), column_index_meta.bloom_filter_index()); RETURN_IF_ERROR(bf_index_reader.load(!config::disable_pk_storage_page_cache, false, - _pk_index_load_stats)); + pk_index_load_stats)); std::unique_ptr bf_iter; - RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter)); + RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter, pk_index_load_stats)); RETURN_IF_ERROR(bf_iter->read_bloom_filter(0, &_bf)); segment_v2::g_pk_total_bloom_filter_num << 1; segment_v2::g_pk_total_bloom_filter_total_bytes << _bf->size(); diff --git a/be/src/olap/primary_key_index.h b/be/src/olap/primary_key_index.h index dcbbc5f30625f4..f74d3e42030f2f 100644 --- a/be/src/olap/primary_key_index.h +++ b/be/src/olap/primary_key_index.h @@ -98,8 +98,7 @@ class PrimaryKeyIndexBuilder { class PrimaryKeyIndexReader { public: - PrimaryKeyIndexReader(OlapReaderStatistics* pk_index_load_stats = nullptr) - : _index_parsed(false), _bf_parsed(false), _pk_index_load_stats(pk_index_load_stats) {} + PrimaryKeyIndexReader() : _index_parsed(false), _bf_parsed(false) {} ~PrimaryKeyIndexReader() { segment_v2::g_pk_total_bloom_filter_num << -static_cast(_bf_num); @@ -109,12 +108,14 @@ class PrimaryKeyIndexReader { } Status parse_index(io::FileReaderSPtr file_reader, - const segment_v2::PrimaryKeyIndexMetaPB& meta); + const segment_v2::PrimaryKeyIndexMetaPB& meta, + OlapReaderStatistics* pk_index_load_stats); - Status parse_bf(io::FileReaderSPtr file_reader, const segment_v2::PrimaryKeyIndexMetaPB& meta); + Status parse_bf(io::FileReaderSPtr file_reader, const segment_v2::PrimaryKeyIndexMetaPB& meta, + OlapReaderStatistics* pk_index_load_stats); Status new_iterator(std::unique_ptr* index_iterator, - OlapReaderStatistics* stats = nullptr) const { + OlapReaderStatistics* stats) const { DCHECK(_index_parsed); index_iterator->reset(new segment_v2::IndexedColumnIterator(_index_reader.get(), stats)); return Status::OK(); @@ -155,7 +156,6 @@ class PrimaryKeyIndexReader { std::unique_ptr _bf; size_t _bf_num = 0; uint64 _bf_bytes = 0; - OlapReaderStatistics* _pk_index_load_stats = nullptr; }; } // namespace doris diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp index 917356f486be6e..3bf6d6ca80ae30 100644 --- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp @@ -34,25 +34,26 @@ namespace segment_v2 { Status BloomFilterIndexReader::load(bool use_page_cache, bool kept_in_memory, OlapReaderStatistics* index_load_stats) { // TODO yyq: implement a new once flag to avoid status construct. - _index_load_stats = index_load_stats; - return _load_once.call([this, use_page_cache, kept_in_memory] { - return _load(use_page_cache, kept_in_memory); + return _load_once.call([this, use_page_cache, kept_in_memory, index_load_stats] { + return _load(use_page_cache, kept_in_memory, index_load_stats); }); } -Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory) { +Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory, + OlapReaderStatistics* index_load_stats) { const IndexedColumnMetaPB& bf_index_meta = _bloom_filter_index_meta->bloom_filter(); _bloom_filter_reader.reset(new IndexedColumnReader(_file_reader, bf_index_meta)); - RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache, kept_in_memory, _index_load_stats)); + RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache, kept_in_memory, index_load_stats)); return Status::OK(); } -Status BloomFilterIndexReader::new_iterator(std::unique_ptr* iterator) { +Status BloomFilterIndexReader::new_iterator(std::unique_ptr* iterator, + OlapReaderStatistics* index_load_stats) { DBUG_EXECUTE_IF("BloomFilterIndexReader::new_iterator.fail", { return Status::InternalError("new_iterator for bloom filter index failed"); }); - iterator->reset(new BloomFilterIndexIterator(this)); + iterator->reset(new BloomFilterIndexIterator(this, index_load_stats)); return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h index effaa876e1c0fd..27643f74d86de9 100644 --- a/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h +++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h @@ -48,17 +48,18 @@ class BloomFilterIndexReader { } Status load(bool use_page_cache, bool kept_in_memory, - OlapReaderStatistics* _bf_index_load_stats = nullptr); + OlapReaderStatistics* bf_index_load_stats); BloomFilterAlgorithmPB algorithm() { return _bloom_filter_index_meta->algorithm(); } // create a new column iterator. - Status new_iterator(std::unique_ptr* iterator); + Status new_iterator(std::unique_ptr* iterator, + OlapReaderStatistics* index_load_stats); const TypeInfo* type_info() const { return _type_info; } private: - Status _load(bool use_page_cache, bool kept_in_memory); + Status _load(bool use_page_cache, bool kept_in_memory, OlapReaderStatistics* index_load_stats); private: friend class BloomFilterIndexIterator; @@ -68,13 +69,12 @@ class BloomFilterIndexReader { const TypeInfo* _type_info = nullptr; std::unique_ptr _bloom_filter_index_meta = nullptr; std::unique_ptr _bloom_filter_reader; - OlapReaderStatistics* _index_load_stats = nullptr; }; class BloomFilterIndexIterator { public: - explicit BloomFilterIndexIterator(BloomFilterIndexReader* reader) - : _reader(reader), _bloom_filter_iter(reader->_bloom_filter_reader.get()) {} + explicit BloomFilterIndexIterator(BloomFilterIndexReader* reader, OlapReaderStatistics* stats) + : _reader(reader), _bloom_filter_iter(reader->_bloom_filter_reader.get(), stats) {} // Read bloom filter at the given ordinal into `bf`. Status read_bloom_filter(rowid_t ordinal, std::unique_ptr* bf); diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index f44885325e4a71..3a74405fe8a264 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -366,10 +366,12 @@ Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const Pag Status ColumnReader::get_row_ranges_by_zone_map( const AndBlockColumnPredicate* col_predicates, - const std::vector* delete_predicates, RowRanges* row_ranges) { + const std::vector* delete_predicates, RowRanges* row_ranges, + const ColumnIteratorOptions& iter_opts) { std::vector page_indexes; - RETURN_IF_ERROR(_get_filtered_pages(col_predicates, delete_predicates, &page_indexes)); - RETURN_IF_ERROR(_calculate_row_ranges(page_indexes, row_ranges)); + RETURN_IF_ERROR( + _get_filtered_pages(col_predicates, delete_predicates, &page_indexes, iter_opts)); + RETURN_IF_ERROR(_calculate_row_ranges(page_indexes, row_ranges, iter_opts)); return Status::OK(); } @@ -506,8 +508,8 @@ bool ColumnReader::_zone_map_match_condition(const ZoneMapPB& zone_map, Status ColumnReader::_get_filtered_pages( const AndBlockColumnPredicate* col_predicates, const std::vector* delete_predicates, - std::vector* page_indexes) { - RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache, _opts.kept_in_memory)); + std::vector* page_indexes, const ColumnIteratorOptions& iter_opts) { + RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts)); FieldType type = _type_info->type(); const std::vector& zone_maps = _zone_map_index->page_zone_maps(); @@ -545,9 +547,10 @@ Status ColumnReader::_get_filtered_pages( } Status ColumnReader::_calculate_row_ranges(const std::vector& page_indexes, - RowRanges* row_ranges) { + RowRanges* row_ranges, + const ColumnIteratorOptions& iter_opts) { row_ranges->clear(); - RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory)); + RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts)); for (auto i : page_indexes) { ordinal_t page_first_id = _ordinal_index->get_first_ordinal(i); ordinal_t page_last_id = _ordinal_index->get_last_ordinal(i); @@ -558,12 +561,14 @@ Status ColumnReader::_calculate_row_ranges(const std::vector& page_ind } Status ColumnReader::get_row_ranges_by_bloom_filter(const AndBlockColumnPredicate* col_predicates, - RowRanges* row_ranges) { - RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory)); - RETURN_IF_ERROR(_load_bloom_filter_index(_use_index_page_cache, _opts.kept_in_memory)); + RowRanges* row_ranges, + const ColumnIteratorOptions& iter_opts) { + RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts)); + RETURN_IF_ERROR( + _load_bloom_filter_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts)); RowRanges bf_row_ranges; std::unique_ptr bf_iter; - RETURN_IF_ERROR(_bloom_filter_index->new_iterator(&bf_iter)); + RETURN_IF_ERROR(_bloom_filter_index->new_iterator(&bf_iter, iter_opts.stats)); size_t range_size = row_ranges->range_size(); // get covered page ids std::set page_ids; @@ -590,13 +595,15 @@ Status ColumnReader::get_row_ranges_by_bloom_filter(const AndBlockColumnPredicat return Status::OK(); } -Status ColumnReader::_load_ordinal_index(bool use_page_cache, bool kept_in_memory) { - return _ordinal_index->load(use_page_cache, kept_in_memory); +Status ColumnReader::_load_ordinal_index(bool use_page_cache, bool kept_in_memory, + const ColumnIteratorOptions& iter_opts) { + return _ordinal_index->load(use_page_cache, kept_in_memory, iter_opts.stats); } -Status ColumnReader::_load_zone_map_index(bool use_page_cache, bool kept_in_memory) { +Status ColumnReader::_load_zone_map_index(bool use_page_cache, bool kept_in_memory, + const ColumnIteratorOptions& iter_opts) { if (_zone_map_index != nullptr) { - return _zone_map_index->load(use_page_cache, kept_in_memory); + return _zone_map_index->load(use_page_cache, kept_in_memory, iter_opts.stats); } return Status::OK(); } @@ -670,15 +677,17 @@ bool ColumnReader::has_bloom_filter_index(bool ngram) const { } } -Status ColumnReader::_load_bloom_filter_index(bool use_page_cache, bool kept_in_memory) { +Status ColumnReader::_load_bloom_filter_index(bool use_page_cache, bool kept_in_memory, + const ColumnIteratorOptions& iter_opts) { if (_bloom_filter_index != nullptr) { - return _bloom_filter_index->load(use_page_cache, kept_in_memory); + return _bloom_filter_index->load(use_page_cache, kept_in_memory, iter_opts.stats); } return Status::OK(); } -Status ColumnReader::seek_to_first(OrdinalPageIndexIterator* iter) { - RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory)); +Status ColumnReader::seek_to_first(OrdinalPageIndexIterator* iter, + const ColumnIteratorOptions& iter_opts) { + RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts)); *iter = _ordinal_index->begin(); if (!iter->valid()) { return Status::NotFound("Failed to seek to first rowid"); @@ -686,8 +695,9 @@ Status ColumnReader::seek_to_first(OrdinalPageIndexIterator* iter) { return Status::OK(); } -Status ColumnReader::seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterator* iter) { - RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory)); +Status ColumnReader::seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterator* iter, + const ColumnIteratorOptions& iter_opts) { + RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts)); *iter = _ordinal_index->seek_at_or_before(ordinal); if (!iter->valid()) { return Status::NotFound("Failed to seek to ordinal {}, ", ordinal); @@ -1161,7 +1171,7 @@ Status FileColumnIterator::init(const ColumnIteratorOptions& opts) { FileColumnIterator::~FileColumnIterator() = default; Status FileColumnIterator::seek_to_first() { - RETURN_IF_ERROR(_reader->seek_to_first(&_page_iter)); + RETURN_IF_ERROR(_reader->seek_to_first(&_page_iter, _opts)); RETURN_IF_ERROR(_read_data_page(_page_iter)); _seek_to_pos_in_page(&_page, 0); @@ -1172,7 +1182,7 @@ Status FileColumnIterator::seek_to_first() { Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) { // if current page contains this row, we don't need to seek if (!_page || !_page.contains(ord) || !_page_iter.valid()) { - RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter)); + RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter, _opts)); RETURN_IF_ERROR(_read_data_page(_page_iter)); } _seek_to_pos_in_page(&_page, ord - _page.first_ordinal); @@ -1420,8 +1430,8 @@ Status FileColumnIterator::get_row_ranges_by_zone_map( const AndBlockColumnPredicate* col_predicates, const std::vector* delete_predicates, RowRanges* row_ranges) { if (_reader->has_zone_map()) { - RETURN_IF_ERROR( - _reader->get_row_ranges_by_zone_map(col_predicates, delete_predicates, row_ranges)); + RETURN_IF_ERROR(_reader->get_row_ranges_by_zone_map(col_predicates, delete_predicates, + row_ranges, _opts)); } return Status::OK(); } @@ -1430,7 +1440,7 @@ Status FileColumnIterator::get_row_ranges_by_bloom_filter( const AndBlockColumnPredicate* col_predicates, RowRanges* row_ranges) { if ((col_predicates->can_do_bloom_filter(false) && _reader->has_bloom_filter_index(false)) || (col_predicates->can_do_bloom_filter(true) && _reader->has_bloom_filter_index(true))) { - RETURN_IF_ERROR(_reader->get_row_ranges_by_bloom_filter(col_predicates, row_ranges)); + RETURN_IF_ERROR(_reader->get_row_ranges_by_bloom_filter(col_predicates, row_ranges, _opts)); } return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index 187490d06fbfbb..5fc31d7c03a1b6 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -149,8 +149,9 @@ class ColumnReader { std::unique_ptr* iterator); // Seek to the first entry in the column. - Status seek_to_first(OrdinalPageIndexIterator* iter); - Status seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterator* iter); + Status seek_to_first(OrdinalPageIndexIterator* iter, const ColumnIteratorOptions& iter_opts); + Status seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterator* iter, + const ColumnIteratorOptions& iter_opts); // read a page from file into a page handle Status read_page(const ColumnIteratorOptions& iter_opts, const PagePointer& pp, @@ -176,11 +177,13 @@ class ColumnReader { // - delete_condition is a delete predicate of one version Status get_row_ranges_by_zone_map(const AndBlockColumnPredicate* col_predicates, const std::vector* delete_predicates, - RowRanges* row_ranges); + RowRanges* row_ranges, + const ColumnIteratorOptions& iter_opts); // get row ranges with bloom filter index Status get_row_ranges_by_bloom_filter(const AndBlockColumnPredicate* col_predicates, - RowRanges* row_ranges); + RowRanges* row_ranges, + const ColumnIteratorOptions& iter_opts); PagePointer get_dict_page_pointer() const { return _meta_dict_page; } @@ -220,13 +223,16 @@ class ColumnReader { return Status::OK(); } - [[nodiscard]] Status _load_zone_map_index(bool use_page_cache, bool kept_in_memory); - [[nodiscard]] Status _load_ordinal_index(bool use_page_cache, bool kept_in_memory); + [[nodiscard]] Status _load_zone_map_index(bool use_page_cache, bool kept_in_memory, + const ColumnIteratorOptions& iter_opts); + [[nodiscard]] Status _load_ordinal_index(bool use_page_cache, bool kept_in_memory, + const ColumnIteratorOptions& iter_opts); [[nodiscard]] Status _load_bitmap_index(bool use_page_cache, bool kept_in_memory); [[nodiscard]] Status _load_inverted_index_index( std::shared_ptr index_file_reader, const TabletIndex* index_meta); - [[nodiscard]] Status _load_bloom_filter_index(bool use_page_cache, bool kept_in_memory); + [[nodiscard]] Status _load_bloom_filter_index(bool use_page_cache, bool kept_in_memory, + const ColumnIteratorOptions& iter_opts); bool _zone_map_match_condition(const ZoneMapPB& zone_map, WrapperField* min_value_container, WrapperField* max_value_container, @@ -240,9 +246,11 @@ class ColumnReader { Status _get_filtered_pages(const AndBlockColumnPredicate* col_predicates, const std::vector* delete_predicates, - std::vector* page_indexes); + std::vector* page_indexes, + const ColumnIteratorOptions& iter_opts); - Status _calculate_row_ranges(const std::vector& page_indexes, RowRanges* row_ranges); + Status _calculate_row_ranges(const std::vector& page_indexes, RowRanges* row_ranges, + const ColumnIteratorOptions& iter_opts); private: int64_t _meta_length; diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp index 4ed98fd9a6b968..8a81ff1469ad39 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp @@ -64,7 +64,6 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory, OlapReaderStatistics* index_load_stats) { _use_page_cache = use_page_cache; _kept_in_memory = kept_in_memory; - _index_load_stats = index_load_stats; _type_info = get_scalar_type_info((FieldType)_meta.data_type()); if (_type_info == nullptr) { @@ -79,7 +78,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory, _sole_data_page = PagePointer(_meta.ordinal_index_meta().root_page()); } else { RETURN_IF_ERROR(load_index_page(_meta.ordinal_index_meta().root_page(), - &_ordinal_index_page_handle, &_ordinal_index_reader)); + &_ordinal_index_page_handle, &_ordinal_index_reader, + index_load_stats)); _has_index_page = true; } } @@ -90,7 +90,8 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory, _sole_data_page = PagePointer(_meta.value_index_meta().root_page()); } else { RETURN_IF_ERROR(load_index_page(_meta.value_index_meta().root_page(), - &_value_index_page_handle, &_value_index_reader)); + &_value_index_page_handle, &_value_index_reader, + index_load_stats)); _has_index_page = true; } } @@ -101,13 +102,14 @@ Status IndexedColumnReader::load(bool use_page_cache, bool kept_in_memory, } Status IndexedColumnReader::load_index_page(const PagePointerPB& pp, PageHandle* handle, - IndexPageReader* reader) { + IndexPageReader* reader, + OlapReaderStatistics* index_load_stats) { Slice body; PageFooterPB footer; BlockCompressionCodec* local_compress_codec; RETURN_IF_ERROR(get_block_compression_codec(_meta.compression(), &local_compress_codec)); RETURN_IF_ERROR(read_page(PagePointer(pp), handle, &body, &footer, INDEX_PAGE, - local_compress_codec, false, _index_load_stats)); + local_compress_codec, false, index_load_stats)); RETURN_IF_ERROR(reader->parse(body, footer.index_page_footer())); _mem_size += body.get_size(); return Status::OK(); diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h b/be/src/olap/rowset/segment_v2/indexed_column_reader.h index 6168fba2ed51c2..2691874346d9af 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h +++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h @@ -73,7 +73,8 @@ class IndexedColumnReader { void set_is_pk_index(bool is_pk) { _is_pk_index = is_pk; } private: - Status load_index_page(const PagePointerPB& pp, PageHandle* handle, IndexPageReader* reader); + Status load_index_page(const PagePointerPB& pp, PageHandle* handle, IndexPageReader* reader, + OlapReaderStatistics* index_load_stats); friend class IndexedColumnIterator; @@ -98,7 +99,6 @@ class IndexedColumnReader { const KeyCoder* _value_key_coder = nullptr; uint64_t _mem_size = 0; bool _is_pk_index = false; - OlapReaderStatistics* _index_load_stats = nullptr; }; class IndexedColumnIterator { diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp index 24b2e3379963bc..196c5f45687617 100644 --- a/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp +++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.cpp @@ -71,15 +71,17 @@ Status OrdinalIndexWriter::finish(io::FileWriter* file_writer, ColumnIndexMetaPB return Status::OK(); } -Status OrdinalIndexReader::load(bool use_page_cache, bool kept_in_memory) { +Status OrdinalIndexReader::load(bool use_page_cache, bool kept_in_memory, + OlapReaderStatistics* index_load_stats) { // TODO yyq: implement a new once flag to avoid status construct. - return _load_once.call([this, use_page_cache, kept_in_memory] { - return _load(use_page_cache, kept_in_memory, std::move(_meta_pb)); + return _load_once.call([this, use_page_cache, kept_in_memory, index_load_stats] { + return _load(use_page_cache, kept_in_memory, std::move(_meta_pb), index_load_stats); }); } Status OrdinalIndexReader::_load(bool use_page_cache, bool kept_in_memory, - std::unique_ptr index_meta) { + std::unique_ptr index_meta, + OlapReaderStatistics* stats) { if (index_meta->root_page().is_root_data_page()) { // only one data page, no index page _num_pages = 1; @@ -90,6 +92,7 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool kept_in_memory, } // need to read index page OlapReaderStatistics tmp_stats; + OlapReaderStatistics* stats_ptr = stats != nullptr ? stats : &tmp_stats; PageReadOptions opts { .use_page_cache = use_page_cache, .kept_in_memory = kept_in_memory, @@ -98,8 +101,9 @@ Status OrdinalIndexReader::_load(bool use_page_cache, bool kept_in_memory, .page_pointer = PagePointer(index_meta->root_page().root_page()), // ordinal index page uses NO_COMPRESSION right now .codec = nullptr, - .stats = &tmp_stats, - .io_ctx = io::IOContext {.is_index_data = true}, + .stats = stats_ptr, + .io_ctx = io::IOContext {.is_index_data = true, + .file_cache_stats = &stats_ptr->file_cache_stats}, }; // read index page diff --git a/be/src/olap/rowset/segment_v2/ordinal_page_index.h b/be/src/olap/rowset/segment_v2/ordinal_page_index.h index 8f9e0afe1bf49b..68a84768558370 100644 --- a/be/src/olap/rowset/segment_v2/ordinal_page_index.h +++ b/be/src/olap/rowset/segment_v2/ordinal_page_index.h @@ -75,7 +75,7 @@ class OrdinalIndexReader { virtual ~OrdinalIndexReader(); // load and parse the index page into memory - Status load(bool use_page_cache, bool kept_in_memory); + Status load(bool use_page_cache, bool kept_in_memory, OlapReaderStatistics* index_load_stats); // the returned iter points to the largest element which is less than `ordinal`, // or points to the first element if all elements are greater than `ordinal`, @@ -94,7 +94,8 @@ class OrdinalIndexReader { private: Status _load(bool use_page_cache, bool kept_in_memory, - std::unique_ptr index_meta); + std::unique_ptr index_meta, + OlapReaderStatistics* index_load_stats); private: friend OrdinalPageIndexIterator; diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 2f43641b778969..bc0e6e671e2d30 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -274,7 +274,7 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o } } - RETURN_IF_ERROR(load_index()); + RETURN_IF_ERROR(load_index(read_options.stats)); if (read_options.delete_condition_predicates->num_of_column_predicate() == 0 && read_options.push_down_agg_type_opt != TPushAggOp::NONE && read_options.push_down_agg_type_opt != TPushAggOp::COUNT_ON_INDEX) { @@ -457,7 +457,7 @@ Status Segment::_parse_footer(SegmentFooterPB* footer) { return Status::OK(); } -Status Segment::_load_pk_bloom_filter() { +Status Segment::_load_pk_bloom_filter(OlapReaderStatistics* stats) { #ifdef BE_TEST if (_pk_index_meta == nullptr) { // for BE UT "segment_cache_test" @@ -471,30 +471,30 @@ Status Segment::_load_pk_bloom_filter() { DCHECK(_pk_index_meta != nullptr); DCHECK(_pk_index_reader != nullptr); - return _load_pk_bf_once.call([this] { - RETURN_IF_ERROR(_pk_index_reader->parse_bf(_file_reader, *_pk_index_meta)); + return _load_pk_bf_once.call([this, stats] { + RETURN_IF_ERROR(_pk_index_reader->parse_bf(_file_reader, *_pk_index_meta, stats)); // _meta_mem_usage += _pk_index_reader->get_bf_memory_size(); return Status::OK(); }); } Status Segment::load_pk_index_and_bf(OlapReaderStatistics* index_load_stats) { - _pk_index_load_stats = index_load_stats; - RETURN_IF_ERROR(load_index()); - RETURN_IF_ERROR(_load_pk_bloom_filter()); + RETURN_IF_ERROR(load_index(index_load_stats)); + RETURN_IF_ERROR(_load_pk_bloom_filter(index_load_stats)); return Status::OK(); } -Status Segment::load_index() { - return _load_index_once.call([this] { +Status Segment::load_index(OlapReaderStatistics* stats) { + return _load_index_once.call([this, stats] { if (_tablet_schema->keys_type() == UNIQUE_KEYS && _pk_index_meta != nullptr) { - _pk_index_reader = std::make_unique(_pk_index_load_stats); - RETURN_IF_ERROR(_pk_index_reader->parse_index(_file_reader, *_pk_index_meta)); + _pk_index_reader = std::make_unique(); + RETURN_IF_ERROR(_pk_index_reader->parse_index(_file_reader, *_pk_index_meta, stats)); // _meta_mem_usage += _pk_index_reader->get_memory_size(); return Status::OK(); } else { // read and parse short key index page OlapReaderStatistics tmp_stats; + OlapReaderStatistics* stats_ptr = stats != nullptr ? stats : &tmp_stats; PageReadOptions opts { .use_page_cache = true, .type = INDEX_PAGE, @@ -503,7 +503,8 @@ Status Segment::load_index() { // short key index page uses NO_COMPRESSION for now .codec = nullptr, .stats = &tmp_stats, - .io_ctx = io::IOContext {.is_index_data = true}, + .io_ctx = io::IOContext {.is_index_data = true, + .file_cache_stats = &stats_ptr->file_cache_stats}, }; Slice body; PageFooterPB footer; @@ -949,7 +950,7 @@ Status Segment::new_inverted_index_iterator(const TabletColumn& tablet_column, Status Segment::lookup_row_key(const Slice& key, const TabletSchema* latest_schema, bool with_seq_col, bool with_rowid, RowLocation* row_location, OlapReaderStatistics* stats) { - RETURN_IF_ERROR(load_pk_index_and_bf()); + RETURN_IF_ERROR(load_pk_index_and_bf(stats)); bool has_seq_col = latest_schema->has_sequence_col(); bool has_rowid = !latest_schema->cluster_key_idxes().empty(); size_t seq_col_length = 0; @@ -1038,9 +1039,10 @@ Status Segment::lookup_row_key(const Slice& key, const TabletSchema* latest_sche } Status Segment::read_key_by_rowid(uint32_t row_id, std::string* key) { - RETURN_IF_ERROR(load_pk_index_and_bf()); + OlapReaderStatistics* null_stat = nullptr; + RETURN_IF_ERROR(load_pk_index_and_bf(null_stat)); std::unique_ptr iter; - RETURN_IF_ERROR(_pk_index_reader->new_iterator(&iter)); + RETURN_IF_ERROR(_pk_index_reader->new_iterator(&iter, null_stat)); auto index_type = vectorized::DataTypeFactory::instance().create_data_type( _pk_index_reader->type_info()->type(), 1, 0); @@ -1095,7 +1097,8 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescripto .use_page_cache = !config::disable_storage_page_cache, .file_reader = file_reader().get(), .stats = &stats, - .io_ctx = io::IOContext {.reader_type = ReaderType::READER_QUERY}, + .io_ctx = io::IOContext {.reader_type = ReaderType::READER_QUERY, + .file_cache_stats = &stats.file_cache_stats}, }; std::vector single_row_loc {row_id}; if (!slot->column_paths().empty()) { diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index 103522e13bd03e..bafd88a5f3b8e8 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -131,7 +131,7 @@ class Segment : public std::enable_shared_from_this { Status lookup_row_key(const Slice& key, const TabletSchema* latest_schema, bool with_seq_col, bool with_rowid, RowLocation* row_location, - OlapReaderStatistics* stats = nullptr); + OlapReaderStatistics* stats); Status read_key_by_rowid(uint32_t row_id, std::string* key); @@ -139,9 +139,9 @@ class Segment : public std::enable_shared_from_this { vectorized::MutableColumnPtr& result, OlapReaderStatistics& stats, std::unique_ptr& iterator_hint); - Status load_index(); + Status load_index(OlapReaderStatistics* stats); - Status load_pk_index_and_bf(OlapReaderStatistics* index_load_stats = nullptr); + Status load_pk_index_and_bf(OlapReaderStatistics* stats); void update_healthy_status(Status new_status) { _healthy_status.update(new_status); } // The segment is loaded into SegmentCache and then will load indices, if there are something wrong @@ -220,7 +220,7 @@ class Segment : public std::enable_shared_from_this { Status _open(); Status _parse_footer(SegmentFooterPB* footer); Status _create_column_readers(const SegmentFooterPB& footer); - Status _load_pk_bloom_filter(); + Status _load_pk_bloom_filter(OlapReaderStatistics* stats); ColumnReader* _get_column_reader(const TabletColumn& col); // Get Iterator which will read variant root column and extract with paths and types info @@ -299,7 +299,6 @@ class Segment : public std::enable_shared_from_this { InvertedIndexFileInfo _idx_file_info; int _be_exec_version = BeExecVersionManager::get_newest_version(); - OlapReaderStatistics* _pk_index_load_stats = nullptr; }; } // namespace segment_v2 diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 4fcaac363cad1d..73f842bb8e66c4 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -1175,7 +1175,7 @@ Status SegmentIterator::_lookup_ordinal_from_pk_index(const RowCursor& key, bool bool exact_match = false; std::unique_ptr index_iterator; - RETURN_IF_ERROR(pk_index_reader->new_iterator(&index_iterator)); + RETURN_IF_ERROR(pk_index_reader->new_iterator(&index_iterator, _opts.stats)); Status status = index_iterator->seek_at_or_after(&index_key, &exact_match); if (UNLIKELY(!status.ok())) { diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp b/be/src/olap/rowset/segment_v2/zone_map_index.cpp index 991df2f94755c9..ea7d74e520a475 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp +++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp @@ -142,18 +142,21 @@ Status TypedZoneMapIndexWriter::finish(io::FileWriter* file_writer, return writer.finish(meta->mutable_page_zone_maps()); } -Status ZoneMapIndexReader::load(bool use_page_cache, bool kept_in_memory) { +Status ZoneMapIndexReader::load(bool use_page_cache, bool kept_in_memory, + OlapReaderStatistics* index_load_stats) { // TODO yyq: implement a new once flag to avoid status construct. - return _load_once.call([this, use_page_cache, kept_in_memory] { - return _load(use_page_cache, kept_in_memory, std::move(_page_zone_maps_meta)); + return _load_once.call([this, use_page_cache, kept_in_memory, index_load_stats] { + return _load(use_page_cache, kept_in_memory, std::move(_page_zone_maps_meta), + index_load_stats); }); } Status ZoneMapIndexReader::_load(bool use_page_cache, bool kept_in_memory, - std::unique_ptr page_zone_maps_meta) { + std::unique_ptr page_zone_maps_meta, + OlapReaderStatistics* index_load_stats) { IndexedColumnReader reader(_file_reader, *page_zone_maps_meta); - RETURN_IF_ERROR(reader.load(use_page_cache, kept_in_memory)); - IndexedColumnIterator iter(&reader); + RETURN_IF_ERROR(reader.load(use_page_cache, kept_in_memory, index_load_stats)); + IndexedColumnIterator iter(&reader, index_load_stats); _page_zone_maps.resize(reader.num_values()); diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.h b/be/src/olap/rowset/segment_v2/zone_map_index.h index 923bd2c2046da7..07cf845fd976ab 100644 --- a/be/src/olap/rowset/segment_v2/zone_map_index.h +++ b/be/src/olap/rowset/segment_v2/zone_map_index.h @@ -154,14 +154,16 @@ class ZoneMapIndexReader { virtual ~ZoneMapIndexReader(); // load all page zone maps into memory - Status load(bool use_page_cache, bool kept_in_memory); + Status load(bool use_page_cache, bool kept_in_memory, + OlapReaderStatistics* index_load_stats = nullptr); const std::vector& page_zone_maps() const { return _page_zone_maps; } int32_t num_pages() const { return _page_zone_maps.size(); } private: - Status _load(bool use_page_cache, bool kept_in_memory, std::unique_ptr); + Status _load(bool use_page_cache, bool kept_in_memory, std::unique_ptr, + OlapReaderStatistics* index_load_stats); private: DorisCallOnce _load_once; diff --git a/be/test/olap/date_bloom_filter_test.cpp b/be/test/olap/date_bloom_filter_test.cpp index 1678f5ee71a9e1..e4b9293021aecd 100644 --- a/be/test/olap/date_bloom_filter_test.cpp +++ b/be/test/olap/date_bloom_filter_test.cpp @@ -155,8 +155,8 @@ TEST_F(DateBloomFilterTest, query_index_test) { { const auto& reader = segment->_column_readers[0]; std::unique_ptr bf_iter; - EXPECT_TRUE(reader->_bloom_filter_index->load(true, true).ok()); - EXPECT_TRUE(reader->_bloom_filter_index->new_iterator(&bf_iter).ok()); + EXPECT_TRUE(reader->_bloom_filter_index->load(true, true, nullptr).ok()); + EXPECT_TRUE(reader->_bloom_filter_index->new_iterator(&bf_iter, nullptr).ok()); std::unique_ptr bf; EXPECT_TRUE(bf_iter->read_bloom_filter(0, &bf).ok()); auto test = [&](const std::string& query_string, bool result) { @@ -174,8 +174,8 @@ TEST_F(DateBloomFilterTest, query_index_test) { { const auto& reader = segment->_column_readers[1]; std::unique_ptr bf_iter; - EXPECT_TRUE(reader->_bloom_filter_index->load(true, true).ok()); - EXPECT_TRUE(reader->_bloom_filter_index->new_iterator(&bf_iter).ok()); + EXPECT_TRUE(reader->_bloom_filter_index->load(true, true, nullptr).ok()); + EXPECT_TRUE(reader->_bloom_filter_index->new_iterator(&bf_iter, nullptr).ok()); std::unique_ptr bf; EXPECT_TRUE(bf_iter->read_bloom_filter(0, &bf).ok()); auto test = [&](const std::string& query_string, bool result) { diff --git a/be/test/olap/primary_key_index_test.cpp b/be/test/olap/primary_key_index_test.cpp index 72aae56cd0938f..9407be938867ec 100644 --- a/be/test/olap/primary_key_index_test.cpp +++ b/be/test/olap/primary_key_index_test.cpp @@ -80,12 +80,12 @@ TEST_F(PrimaryKeyIndexTest, builder) { PrimaryKeyIndexReader index_reader; io::FileReaderSPtr file_reader; EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); - EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta).ok()); - EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta).ok()); + EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta, nullptr).ok()); + EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta, nullptr).ok()); EXPECT_EQ(num_rows, index_reader.num_rows()); std::unique_ptr index_iterator; - EXPECT_TRUE(index_reader.new_iterator(&index_iterator).ok()); + EXPECT_TRUE(index_reader.new_iterator(&index_iterator, nullptr).ok()); bool exact_match = false; uint32_t row_id; for (size_t i = 0; i < keys.size(); i++) { @@ -142,7 +142,7 @@ TEST_F(PrimaryKeyIndexTest, builder) { int batch_size = 1024; while (remaining > 0) { std::unique_ptr iter; - EXPECT_TRUE(index_reader.new_iterator(&iter).ok()); + EXPECT_TRUE(index_reader.new_iterator(&iter, nullptr).ok()); size_t num_to_read = std::min(batch_size, remaining); auto index_type = vectorized::DataTypeFactory::instance().create_data_type( @@ -199,12 +199,12 @@ TEST_F(PrimaryKeyIndexTest, multiple_pages) { PrimaryKeyIndexReader index_reader; io::FileReaderSPtr file_reader; EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); - EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta).ok()); - EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta).ok()); + EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta, nullptr).ok()); + EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta, nullptr).ok()); EXPECT_EQ(num_rows, index_reader.num_rows()); std::unique_ptr index_iterator; - EXPECT_TRUE(index_reader.new_iterator(&index_iterator).ok()); + EXPECT_TRUE(index_reader.new_iterator(&index_iterator, nullptr).ok()); bool exact_match = false; uint32_t row_id; for (size_t i = 0; i < keys.size(); i++) { @@ -283,12 +283,12 @@ TEST_F(PrimaryKeyIndexTest, single_page) { PrimaryKeyIndexReader index_reader; io::FileReaderSPtr file_reader; EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); - EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta).ok()); - EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta).ok()); + EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta, nullptr).ok()); + EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta, nullptr).ok()); EXPECT_EQ(num_rows, index_reader.num_rows()); std::unique_ptr index_iterator; - EXPECT_TRUE(index_reader.new_iterator(&index_iterator).ok()); + EXPECT_TRUE(index_reader.new_iterator(&index_iterator, nullptr).ok()); bool exact_match = false; uint32_t row_id; for (size_t i = 0; i < keys.size(); i++) { diff --git a/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp index 258dd9a5ff8b51..1f5a512688a54e 100644 --- a/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp +++ b/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp @@ -102,10 +102,10 @@ void get_bloom_filter_reader_iter(const std::string& file_name, const ColumnInde io::FileReaderSPtr file_reader; ASSERT_EQ(io::global_local_filesystem()->open_file(fname, &file_reader), Status::OK()); *reader = new BloomFilterIndexReader(std::move(file_reader), meta.bloom_filter_index()); - auto st = (*reader)->load(true, false); + auto st = (*reader)->load(true, false, nullptr); EXPECT_TRUE(st.ok()); - st = (*reader)->new_iterator(iter); + st = (*reader)->new_iterator(iter, nullptr); EXPECT_TRUE(st.ok()); } diff --git a/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp b/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp index 33848c5959cfaa..ffd9c92ee0272e 100644 --- a/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp +++ b/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp @@ -74,7 +74,7 @@ TEST_F(OrdinalPageIndexTest, normal) { io::FileReaderSPtr file_reader; EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); OrdinalIndexReader index(file_reader, 16 * 1024 * 4096 + 1, index_meta.ordinal_index()); - EXPECT_TRUE(index.load(true, false).ok()); + EXPECT_TRUE(index.load(true, false, nullptr).ok()); EXPECT_EQ(16 * 1024, index.num_data_pages()); EXPECT_EQ(1, index.get_first_ordinal(0)); EXPECT_EQ(4096, index.get_last_ordinal(0)); @@ -128,7 +128,7 @@ TEST_F(OrdinalPageIndexTest, one_data_page) { } OrdinalIndexReader index(nullptr, num_values, index_meta.ordinal_index()); - EXPECT_TRUE(index.load(true, false).ok()); + EXPECT_TRUE(index.load(true, false, nullptr).ok()); EXPECT_EQ(1, index.num_data_pages()); EXPECT_EQ(0, index.get_first_ordinal(0)); EXPECT_EQ(num_values - 1, index.get_last_ordinal(0)); diff --git a/be/test/olap/segment_cache_test.cpp b/be/test/olap/segment_cache_test.cpp index b226bc6c2292e5..c527ffddd424b9 100644 --- a/be/test/olap/segment_cache_test.cpp +++ b/be/test/olap/segment_cache_test.cpp @@ -323,7 +323,7 @@ TEST_F(SegmentCacheTest, vec_sequence_col) { segment_v2::SegmentSharedPtr segment_ptr = handle.get_segments()[0]; // load index and bf second - res = segment_ptr->load_pk_index_and_bf(); + res = segment_ptr->load_pk_index_and_bf(nullptr); ASSERT_TRUE(res.ok()); // check cache mem usage equals to segment mem usage From f26107d6acc738c98da0f916c53394d70fd5d604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Sat, 21 Dec 2024 16:40:27 +0800 Subject: [PATCH 2/2] fix formatter --- be/src/olap/rowset/segment_v2/segment.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index bafd88a5f3b8e8..96409bff343d61 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -130,8 +130,7 @@ class Segment : public std::enable_shared_from_this { } Status lookup_row_key(const Slice& key, const TabletSchema* latest_schema, bool with_seq_col, - bool with_rowid, RowLocation* row_location, - OlapReaderStatistics* stats); + bool with_rowid, RowLocation* row_location, OlapReaderStatistics* stats); Status read_key_by_rowid(uint32_t row_id, std::string* key);