diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index c1a2e3c18b5b66..7e0cf6645d2bff 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -361,10 +361,13 @@ struct OlapReaderStatistics { int64_t inverted_index_query_timer = 0; int64_t inverted_index_query_cache_hit = 0; int64_t inverted_index_query_cache_miss = 0; + int64_t inverted_index_query_null_bitmap_timer = 0; int64_t inverted_index_query_bitmap_copy_timer = 0; int64_t inverted_index_query_bitmap_op_timer = 0; int64_t inverted_index_searcher_open_timer = 0; int64_t inverted_index_searcher_search_timer = 0; + int64_t inverted_index_searcher_cache_hit = 0; + int64_t inverted_index_searcher_cache_miss = 0; int64_t output_index_result_column_timer = 0; // number of segment filtered by column stat when creating seg iterator diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index d89d089de3bf0f..2b90b04ea940eb 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -187,8 +187,10 @@ void InvertedIndexReader::get_analyse_result(std::vector& analyse_r } } -Status InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle, +Status InvertedIndexReader::read_null_bitmap(OlapReaderStatistics* stats, + InvertedIndexQueryCacheHandle* cache_handle, lucene::store::Directory* dir) { + SCOPED_RAW_TIMER(&stats->inverted_index_query_null_bitmap_timer); lucene::store::IndexInput* null_bitmap_in = nullptr; bool owned_dir = false; try { @@ -244,9 +246,11 @@ Status InvertedIndexReader::handle_searcher_cache( InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); if (InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key, inverted_index_cache_handle)) { + stats->inverted_index_searcher_cache_hit++; return Status::OK(); } else { // searcher cache miss + stats->inverted_index_searcher_cache_miss++; auto mem_tracker = std::make_unique("InvertedIndexSearcherCacheWithRead"); SCOPED_RAW_TIMER(&stats->inverted_index_searcher_open_timer); IndexSearcherPtr searcher; @@ -256,7 +260,7 @@ Status InvertedIndexReader::handle_searcher_cache( // to avoid open directory additionally for null_bitmap // TODO: handle null bitmap procedure in new format. InvertedIndexQueryCacheHandle null_bitmap_cache_handle; - static_cast(read_null_bitmap(&null_bitmap_cache_handle, dir.get())); + static_cast(read_null_bitmap(stats, &null_bitmap_cache_handle, dir.get())); RETURN_IF_ERROR(create_index_searcher(dir.release(), &searcher, mem_tracker.get(), type())); auto* cache_value = new InvertedIndexSearcherCache::CacheValue( std::move(searcher), mem_tracker->consumption(), UnixMillis()); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h index 90df92f0728a9b..249169ca7505ca 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h @@ -94,7 +94,8 @@ class InvertedIndexReader : public std::enable_shared_from_thisread_null_bitmap(cache_handle, dir); + return _reader->read_null_bitmap(_stats, cache_handle, dir); } [[nodiscard]] InvertedIndexReaderType get_inverted_index_reader_type() const; diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp b/be/src/pipeline/exec/olap_scan_operator.cpp index bee550f1db5291..95bfd3980417b9 100644 --- a/be/src/pipeline/exec/olap_scan_operator.cpp +++ b/be/src/pipeline/exec/olap_scan_operator.cpp @@ -129,6 +129,8 @@ Status OlapScanLocalState::_init_profile() { _inverted_index_query_cache_miss_counter = ADD_COUNTER(_segment_profile, "InvertedIndexQueryCacheMiss", TUnit::UNIT); _inverted_index_query_timer = ADD_TIMER(_segment_profile, "InvertedIndexQueryTime"); + _inverted_index_query_null_bitmap_timer = + ADD_TIMER(_segment_profile, "InvertedIndexQueryNullBitmapTime"); _inverted_index_query_bitmap_copy_timer = ADD_TIMER(_segment_profile, "InvertedIndexQueryBitmapCopyTime"); _inverted_index_query_bitmap_op_timer = @@ -137,6 +139,10 @@ Status OlapScanLocalState::_init_profile() { ADD_TIMER(_segment_profile, "InvertedIndexSearcherOpenTime"); _inverted_index_searcher_search_timer = ADD_TIMER(_segment_profile, "InvertedIndexSearcherSearchTime"); + _inverted_index_searcher_cache_hit_counter = + ADD_COUNTER(_segment_profile, "InvertedIndexSearcherCacheHit", TUnit::UNIT); + _inverted_index_searcher_cache_miss_counter = + ADD_COUNTER(_segment_profile, "InvertedIndexSearcherCacheMiss", TUnit::UNIT); _output_index_result_column_timer = ADD_TIMER(_segment_profile, "OutputIndexResultColumnTimer"); diff --git a/be/src/pipeline/exec/olap_scan_operator.h b/be/src/pipeline/exec/olap_scan_operator.h index ca98b17118999f..83f838dd0fc47c 100644 --- a/be/src/pipeline/exec/olap_scan_operator.h +++ b/be/src/pipeline/exec/olap_scan_operator.h @@ -174,6 +174,7 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _inverted_index_filter_counter = nullptr; RuntimeProfile::Counter* _inverted_index_filter_timer = nullptr; + RuntimeProfile::Counter* _inverted_index_query_null_bitmap_timer = nullptr; RuntimeProfile::Counter* _inverted_index_query_cache_hit_counter = nullptr; RuntimeProfile::Counter* _inverted_index_query_cache_miss_counter = nullptr; RuntimeProfile::Counter* _inverted_index_query_timer = nullptr; @@ -181,6 +182,8 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _inverted_index_query_bitmap_op_timer = nullptr; RuntimeProfile::Counter* _inverted_index_searcher_open_timer = nullptr; RuntimeProfile::Counter* _inverted_index_searcher_search_timer = nullptr; + RuntimeProfile::Counter* _inverted_index_searcher_cache_hit_counter = nullptr; + RuntimeProfile::Counter* _inverted_index_searcher_cache_miss_counter = nullptr; RuntimeProfile::Counter* _output_index_result_column_timer = nullptr; diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp index d539c0d7864f19..5023dc7639cf85 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.cpp +++ b/be/src/vec/exec/scan/new_olap_scanner.cpp @@ -618,6 +618,8 @@ void NewOlapScanner::_collect_profile_before_close() { COUNTER_UPDATE(Parent->_inverted_index_query_cache_miss_counter, \ stats.inverted_index_query_cache_miss); \ COUNTER_UPDATE(Parent->_inverted_index_query_timer, stats.inverted_index_query_timer); \ + COUNTER_UPDATE(Parent->_inverted_index_query_null_bitmap_timer, \ + stats.inverted_index_query_null_bitmap_timer); \ COUNTER_UPDATE(Parent->_inverted_index_query_bitmap_copy_timer, \ stats.inverted_index_query_bitmap_copy_timer); \ COUNTER_UPDATE(Parent->_inverted_index_query_bitmap_op_timer, \ @@ -626,6 +628,10 @@ void NewOlapScanner::_collect_profile_before_close() { stats.inverted_index_searcher_open_timer); \ COUNTER_UPDATE(Parent->_inverted_index_searcher_search_timer, \ stats.inverted_index_searcher_search_timer); \ + COUNTER_UPDATE(Parent->_inverted_index_searcher_cache_hit_counter, \ + stats.inverted_index_searcher_cache_hit); \ + COUNTER_UPDATE(Parent->_inverted_index_searcher_cache_miss_counter, \ + stats.inverted_index_searcher_cache_miss); \ if (config::enable_file_cache) { \ io::FileCacheProfileReporter cache_profile(Parent->_segment_profile.get()); \ cache_profile.update(&stats.file_cache_stats); \