Skip to content

[opt](index) add more inverted index profile metrics #36696

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xiaokang merged 4 commits into from
Aug 5, 2024
Merged

[opt](index) add more inverted index profile metrics #36696

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2462389
add more inverted index profile metrics
xiaokang Jun 22, 2024
bdd5da7
clang format
xiaokang Jun 22, 2024
7ed5d91
Merge branch 'master' into add_index_profile_master
xiaokang Jul 12, 2024
1c5a2a9
Merge branch 'master' into add_index_profile_master
xiaokang Jul 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions be/src/olap/olap_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,10 +361,13 @@ struct OlapReaderStatistics {
int64_t inverted_index_query_timer = 0;
int64_t inverted_index_query_cache_hit = 0;
int64_t inverted_index_query_cache_miss = 0;
int64_t inverted_index_query_null_bitmap_timer = 0;
int64_t inverted_index_query_bitmap_copy_timer = 0;
int64_t inverted_index_query_bitmap_op_timer = 0;
int64_t inverted_index_searcher_open_timer = 0;
int64_t inverted_index_searcher_search_timer = 0;
int64_t inverted_index_searcher_cache_hit = 0;
int64_t inverted_index_searcher_cache_miss = 0;

int64_t output_index_result_column_timer = 0;
// number of segment filtered by column stat when creating seg iterator
Expand Down
8 changes: 6 additions & 2 deletions be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,10 @@ void InvertedIndexReader::get_analyse_result(std::vector<std::string>& analyse_r
}
}

Status InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
Status InvertedIndexReader::read_null_bitmap(OlapReaderStatistics* stats,
InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir) {
SCOPED_RAW_TIMER(&stats->inverted_index_query_null_bitmap_timer);
lucene::store::IndexInput* null_bitmap_in = nullptr;
bool owned_dir = false;
try {
Expand Down Expand Up @@ -244,9 +246,11 @@ Status InvertedIndexReader::handle_searcher_cache(
InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key);
if (InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key,
inverted_index_cache_handle)) {
stats->inverted_index_searcher_cache_hit++;
return Status::OK();
} else {
// searcher cache miss
stats->inverted_index_searcher_cache_miss++;
auto mem_tracker = std::make_unique<MemTracker>("InvertedIndexSearcherCacheWithRead");
SCOPED_RAW_TIMER(&stats->inverted_index_searcher_open_timer);
IndexSearcherPtr searcher;
Expand All @@ -256,7 +260,7 @@ Status InvertedIndexReader::handle_searcher_cache(
// to avoid open directory additionally for null_bitmap
// TODO: handle null bitmap procedure in new format.
InvertedIndexQueryCacheHandle null_bitmap_cache_handle;
static_cast<void>(read_null_bitmap(&null_bitmap_cache_handle, dir.get()));
static_cast<void>(read_null_bitmap(stats, &null_bitmap_cache_handle, dir.get()));
RETURN_IF_ERROR(create_index_searcher(dir.release(), &searcher, mem_tracker.get(), type()));
auto* cache_value = new InvertedIndexSearcherCache::CacheValue(
std::move(searcher), mem_tracker->consumption(), UnixMillis());
Expand Down
5 changes: 3 additions & 2 deletions be/src/olap/rowset/segment_v2/inverted_index_reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ class InvertedIndexReader : public std::enable_shared_from_this<InvertedIndexRea
const void* query_value, InvertedIndexQueryType query_type,
uint32_t* count) = 0;

Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
Status read_null_bitmap(OlapReaderStatistics* stats,
InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir = nullptr);

virtual InvertedIndexReaderType type() = 0;
Expand Down Expand Up @@ -373,7 +374,7 @@ class InvertedIndexIterator {

Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir = nullptr) {
return _reader->read_null_bitmap(cache_handle, dir);
return _reader->read_null_bitmap(_stats, cache_handle, dir);
}

[[nodiscard]] InvertedIndexReaderType get_inverted_index_reader_type() const;
Expand Down
6 changes: 6 additions & 0 deletions be/src/pipeline/exec/olap_scan_operator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,8 @@ Status OlapScanLocalState::_init_profile() {
_inverted_index_query_cache_miss_counter =
ADD_COUNTER(_segment_profile, "InvertedIndexQueryCacheMiss", TUnit::UNIT);
_inverted_index_query_timer = ADD_TIMER(_segment_profile, "InvertedIndexQueryTime");
_inverted_index_query_null_bitmap_timer =
ADD_TIMER(_segment_profile, "InvertedIndexQueryNullBitmapTime");
_inverted_index_query_bitmap_copy_timer =
ADD_TIMER(_segment_profile, "InvertedIndexQueryBitmapCopyTime");
_inverted_index_query_bitmap_op_timer =
Expand All @@ -137,6 +139,10 @@ Status OlapScanLocalState::_init_profile() {
ADD_TIMER(_segment_profile, "InvertedIndexSearcherOpenTime");
_inverted_index_searcher_search_timer =
ADD_TIMER(_segment_profile, "InvertedIndexSearcherSearchTime");
_inverted_index_searcher_cache_hit_counter =
ADD_COUNTER(_segment_profile, "InvertedIndexSearcherCacheHit", TUnit::UNIT);
_inverted_index_searcher_cache_miss_counter =
ADD_COUNTER(_segment_profile, "InvertedIndexSearcherCacheMiss", TUnit::UNIT);

_output_index_result_column_timer = ADD_TIMER(_segment_profile, "OutputIndexResultColumnTimer");

Expand Down
3 changes: 3 additions & 0 deletions be/src/pipeline/exec/olap_scan_operator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,13 +174,16 @@ class OlapScanLocalState final : public ScanLocalState<OlapScanLocalState> {

RuntimeProfile::Counter* _inverted_index_filter_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_filter_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_query_null_bitmap_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_query_cache_hit_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_query_cache_miss_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_query_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_query_bitmap_copy_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_query_bitmap_op_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_searcher_open_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_searcher_search_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_searcher_cache_hit_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_searcher_cache_miss_counter = nullptr;

RuntimeProfile::Counter* _output_index_result_column_timer = nullptr;

Expand Down
6 changes: 6 additions & 0 deletions be/src/vec/exec/scan/new_olap_scanner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -618,6 +618,8 @@ void NewOlapScanner::_collect_profile_before_close() {
COUNTER_UPDATE(Parent->_inverted_index_query_cache_miss_counter, \
stats.inverted_index_query_cache_miss); \
COUNTER_UPDATE(Parent->_inverted_index_query_timer, stats.inverted_index_query_timer); \
COUNTER_UPDATE(Parent->_inverted_index_query_null_bitmap_timer, \
stats.inverted_index_query_null_bitmap_timer); \
COUNTER_UPDATE(Parent->_inverted_index_query_bitmap_copy_timer, \
stats.inverted_index_query_bitmap_copy_timer); \
COUNTER_UPDATE(Parent->_inverted_index_query_bitmap_op_timer, \
Expand All @@ -626,6 +628,10 @@ void NewOlapScanner::_collect_profile_before_close() {
stats.inverted_index_searcher_open_timer); \
COUNTER_UPDATE(Parent->_inverted_index_searcher_search_timer, \
stats.inverted_index_searcher_search_timer); \
COUNTER_UPDATE(Parent->_inverted_index_searcher_cache_hit_counter, \
stats.inverted_index_searcher_cache_hit); \
COUNTER_UPDATE(Parent->_inverted_index_searcher_cache_miss_counter, \
stats.inverted_index_searcher_cache_miss); \
if (config::enable_file_cache) { \
io::FileCacheProfileReporter cache_profile(Parent->_segment_profile.get()); \
cache_profile.update(&stats.file_cache_stats); \
Expand Down