diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 9f33f363e99bb3..df936f5f4aef7a 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -163,7 +163,8 @@ Status BetaRowset::load_segments(int64_t seg_id_begin, int64_t seg_id_end, return Status::OK(); } -Status BetaRowset::load_segment(int64_t seg_id, segment_v2::SegmentSharedPtr* segment) { +Status BetaRowset::load_segment(int64_t seg_id, segment_v2::SegmentSharedPtr* segment, + OlapReaderStatistics* stats) { auto fs = _rowset_meta->fs(); if (!fs) { return Status::Error("get fs failed"); @@ -181,7 +182,7 @@ Status BetaRowset::load_segment(int64_t seg_id, segment_v2::SegmentSharedPtr* se auto s = segment_v2::Segment::open(fs, seg_path, _rowset_meta->tablet_id(), seg_id, rowset_id(), _schema, reader_options, segment, - _rowset_meta->inverted_index_file_info(seg_id)); + _rowset_meta->inverted_index_file_info(seg_id), stats); if (!s.ok()) { LOG(WARNING) << "failed to open segment. " << seg_path << " under rowset " << rowset_id() << " : " << s.to_string(); diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 0b22d122741770..32d8f3500a8cf4 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -26,6 +26,7 @@ #include #include "common/status.h" +#include "olap/olap_common.h" #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_meta.h" #include "olap/rowset/rowset_reader.h" @@ -76,7 +77,8 @@ class BetaRowset final : public Rowset { Status load_segments(int64_t seg_id_begin, int64_t seg_id_end, std::vector* segments); - Status load_segment(int64_t seg_id, segment_v2::SegmentSharedPtr* segment); + Status load_segment(int64_t seg_id, segment_v2::SegmentSharedPtr* segment, + OlapReaderStatistics* stats = nullptr); Status get_segments_size(std::vector* segments_size); diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index 9a4d71587a02c1..cd3994ec072285 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -231,7 +231,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context SCOPED_RAW_TIMER(&_stats->rowset_reader_load_segments_timer_ns); RETURN_IF_ERROR(SegmentLoader::instance()->load_segments( _rowset, &segment_cache_handle, should_use_cache, - /*need_load_pk_index_and_bf*/ false)); + /*need_load_pk_index_and_bf*/ false, _stats)); } // create iterator for each segment diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 298f4684e84025..86884d7adcec47 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -81,9 +81,9 @@ class InvertedIndexIterator; Status Segment::open(io::FileSystemSPtr fs, const std::string& path, int64_t tablet_id, uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr tablet_schema, const io::FileReaderOptions& reader_options, std::shared_ptr* output, - InvertedIndexFileInfo idx_file_info) { + InvertedIndexFileInfo idx_file_info, OlapReaderStatistics* stats) { auto s = _open(fs, path, segment_id, rowset_id, tablet_schema, reader_options, output, - idx_file_info); + idx_file_info, stats); if (!s.ok()) { if (!config::is_cloud_mode()) { auto res = ExecEnv::get_tablet(tablet_id); @@ -101,14 +101,14 @@ Status Segment::open(io::FileSystemSPtr fs, const std::string& path, int64_t tab Status Segment::_open(io::FileSystemSPtr fs, const std::string& path, uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr tablet_schema, const io::FileReaderOptions& reader_options, std::shared_ptr* output, - InvertedIndexFileInfo idx_file_info) { + InvertedIndexFileInfo idx_file_info, OlapReaderStatistics* stats) { io::FileReaderSPtr file_reader; RETURN_IF_ERROR(fs->open_file(path, &file_reader, &reader_options)); std::shared_ptr segment( new Segment(segment_id, rowset_id, std::move(tablet_schema), idx_file_info)); segment->_fs = fs; segment->_file_reader = std::move(file_reader); - auto st = segment->_open(); + auto st = segment->_open(stats); TEST_INJECTION_POINT_CALLBACK("Segment::open:corruption", &st); if (st.is() && reader_options.cache_type == io::FileCachePolicy::FILE_BLOCK_CACHE) { @@ -121,7 +121,7 @@ Status Segment::_open(io::FileSystemSPtr fs, const std::string& path, uint32_t s RETURN_IF_ERROR(fs->open_file(path, &file_reader, &reader_options)); segment->_file_reader = std::move(file_reader); - st = segment->_open(); + st = segment->_open(stats); TEST_INJECTION_POINT_CALLBACK("Segment::open:corruption1", &st); if (st.is()) { // corrupt again LOG(WARNING) << "failed to try to read remote source file again with cache support," @@ -134,7 +134,7 @@ Status Segment::_open(io::FileSystemSPtr fs, const std::string& path, uint32_t s opt.cache_type = io::FileCachePolicy::NO_CACHE; // skip cache RETURN_IF_ERROR(fs->open_file(path, &file_reader, &opt)); segment->_file_reader = std::move(file_reader); - st = segment->_open(); + st = segment->_open(stats); if (!st.ok()) { LOG(WARNING) << "failed to try to read remote source file directly," << " file path: " << path @@ -176,9 +176,9 @@ void Segment::update_metadata_size() { _tracked_meta_mem_usage = _meta_mem_usage; } -Status Segment::_open() { +Status Segment::_open(OlapReaderStatistics* stats) { _footer_pb = std::make_unique(); - RETURN_IF_ERROR(_parse_footer(_footer_pb.get())); + RETURN_IF_ERROR(_parse_footer(_footer_pb.get(), stats)); _pk_index_meta.reset(_footer_pb->has_primary_key_index_meta() ? new PrimaryKeyIndexMetaPB(_footer_pb->primary_key_index_meta()) : nullptr); @@ -387,7 +387,7 @@ Status Segment::_write_error_file(size_t file_size, size_t offset, size_t bytes_ return Status::OK(); // already exists }; -Status Segment::_parse_footer(SegmentFooterPB* footer) { +Status Segment::_parse_footer(SegmentFooterPB* footer, OlapReaderStatistics* stats) { // Footer := SegmentFooterPB, FooterPBSize(4), FooterPBChecksum(4), MagicNumber(4) auto file_size = _file_reader->size(); if (file_size < 12) { @@ -399,7 +399,8 @@ Status Segment::_parse_footer(SegmentFooterPB* footer) { uint8_t fixed_buf[12]; size_t bytes_read = 0; // TODO(plat1ko): Support session variable `enable_file_cache` - io::IOContext io_ctx {.is_index_data = true}; + io::IOContext io_ctx {.is_index_data = true, + .file_cache_stats = stats ? &stats->file_cache_stats : nullptr}; RETURN_IF_ERROR( _file_reader->read_at(file_size - 12, Slice(fixed_buf, 12), &bytes_read, &io_ctx)); DCHECK_EQ(bytes_read, 12); diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index b8478d60cd5d8d..de024b30a99c67 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -82,7 +82,8 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd static Status open(io::FileSystemSPtr fs, const std::string& path, int64_t tablet_id, uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr tablet_schema, const io::FileReaderOptions& reader_options, - std::shared_ptr* output, InvertedIndexFileInfo idx_file_info = {}); + std::shared_ptr* output, InvertedIndexFileInfo idx_file_info = {}, + OlapReaderStatistics* stats = nullptr); static io::UInt128Wrapper file_cache_key(std::string_view rowset_id, uint32_t seg_id); io::UInt128Wrapper file_cache_key() const { @@ -220,10 +221,11 @@ class Segment : public std::enable_shared_from_this, public MetadataAdd static Status _open(io::FileSystemSPtr fs, const std::string& path, uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr tablet_schema, const io::FileReaderOptions& reader_options, - std::shared_ptr* output, InvertedIndexFileInfo idx_file_info); + std::shared_ptr* output, InvertedIndexFileInfo idx_file_info, + OlapReaderStatistics* stats = nullptr); // open segment file and read the minimum amount of necessary information (footer) - Status _open(); - Status _parse_footer(SegmentFooterPB* footer); + Status _open(OlapReaderStatistics* stats); + Status _parse_footer(SegmentFooterPB* footer, OlapReaderStatistics* stats = nullptr); Status _create_column_readers(const SegmentFooterPB& footer); Status _load_pk_bloom_filter(OlapReaderStatistics* stats); ColumnReader* _get_column_reader(const TabletColumn& col); diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp index 4240f7e250a06b..56af67f487ab4d 100644 --- a/be/src/olap/segment_loader.cpp +++ b/be/src/olap/segment_loader.cpp @@ -71,7 +71,7 @@ Status SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset, } // If the segment is not healthy, then will create a new segment and will replace the unhealthy one in SegmentCache. segment_v2::SegmentSharedPtr segment; - RETURN_IF_ERROR(rowset->load_segment(i, &segment)); + RETURN_IF_ERROR(rowset->load_segment(i, &segment, index_load_stats)); if (need_load_pk_index_and_bf) { RETURN_IF_ERROR(segment->load_pk_index_and_bf(index_load_stats)); }