From 6ba037cf5f7eefcbf68634c70d3631a5e9bdd560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 7 Aug 2024 11:02:43 +0800 Subject: [PATCH 1/2] [Improvement](inverted index) lazy init inverted index file reader --- .../segment_v2/inverted_index_file_reader.cpp | 17 +++++++++++------ .../segment_v2/inverted_index_file_reader.h | 1 + .../segment_v2/inverted_index_file_writer.cpp | 4 +++- .../rowset/segment_v2/inverted_index_reader.cpp | 7 +++++++ be/src/olap/rowset/segment_v2/segment.cpp | 9 +-------- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp index dbd86bb93a511e..09a6a62aaa6115 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp @@ -28,13 +28,18 @@ namespace doris::segment_v2 { Status InvertedIndexFileReader::init(int32_t read_buffer_size, bool open_idx_file_cache) { - _read_buffer_size = read_buffer_size; - _open_idx_file_cache = open_idx_file_cache; - if (_storage_format == InvertedIndexStorageFormatPB::V2) { - return _init_from_v2(read_buffer_size); - } else { - return Status::OK(); + if (!_inited) { + _read_buffer_size = read_buffer_size; + _open_idx_file_cache = open_idx_file_cache; + if (_storage_format == InvertedIndexStorageFormatPB::V2) { + auto st = _init_from_v2(read_buffer_size); + if (!st.ok()) { + return st; + } + } + _inited = true; } + return Status::OK(); } Status InvertedIndexFileReader::_init_from_v2(int32_t read_buffer_size) { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h index fc0dd6e43d1c70..1414f493e4bb2a 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h @@ -79,6 +79,7 @@ class InvertedIndexFileReader { bool _open_idx_file_cache = false; InvertedIndexStorageFormatPB _storage_format; mutable std::shared_mutex _mutex; // Use mutable for const read operations + bool _inited = false; }; } // namespace segment_v2 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp index 3dd842b16d8808..f2ac0e922650c6 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp @@ -115,7 +115,9 @@ Status InvertedIndexFileWriter::close() { } DBUG_EXECUTE_IF("inverted_index_storage_format_must_be_v2", { if (_storage_format != InvertedIndexStorageFormatPB::V2) { - _CLTHROWA(CL_ERR_IO, "inverted index storage format must be v2"); + return Status::Error( + "InvertedIndexFileWriter::close fault injection:inverted index storage format " + "must be v2"); } }) if (_storage_format == InvertedIndexStorageFormatPB::V1) { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index 02339c000806ea..741bb72cfc402b 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -255,6 +255,13 @@ Status InvertedIndexReader::handle_searcher_cache( SCOPED_RAW_TIMER(&stats->inverted_index_searcher_open_timer); IndexSearcherPtr searcher; + bool open_idx_file_cache = true; + auto st = _inverted_index_file_reader->init(config::inverted_index_read_buffer_size, + open_idx_file_cache); + if (!st.ok()) { + LOG(WARNING) << st; + return st; + } auto dir = DORIS_TRY(_inverted_index_file_reader->open(&_index_meta)); // try to reuse index_searcher's directory to read null_bitmap to cache // to avoid open directory additionally for null_bitmap diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 1b31117f126aa2..7a36a7b162c081 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -185,14 +185,7 @@ Status Segment::_open_inverted_index() { std::string {InvertedIndexDescriptor::get_index_file_path_prefix( _file_reader->path().native())}, _tablet_schema->get_inverted_index_storage_format()); - bool open_idx_file_cache = true; - auto st = _inverted_index_file_reader->init(config::inverted_index_read_buffer_size, - open_idx_file_cache); - if (st.is()) { - LOG(INFO) << st; - return Status::OK(); - } - return st; + return Status::OK(); } Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_options, From fcb9d4cf6a0889aa4a7b156ff446fc35461bf012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 7 Aug 2024 14:08:28 +0800 Subject: [PATCH 2/2] fix null bitmap open --- be/src/olap/rowset/segment_v2/inverted_index_reader.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index 741bb72cfc402b..72e373425efdfd 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -205,6 +205,13 @@ Status InvertedIndexReader::read_null_bitmap(OlapReaderStatistics* stats, if (!dir) { // TODO: ugly code here, try to refact. + bool open_idx_file_cache = true; + auto st = _inverted_index_file_reader->init(config::inverted_index_read_buffer_size, + open_idx_file_cache); + if (!st.ok()) { + LOG(WARNING) << st; + return st; + } auto directory = DORIS_TRY(_inverted_index_file_reader->open(&_index_meta)); dir = directory.release(); owned_dir = true;