From 4b7e5cea63c17ea5668c76c2abc9dc2b87540721 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Tue, 6 Aug 2024 14:39:05 +0800 Subject: [PATCH 1/3] [fix](cloud-balance) use right file size instead of block size introduced by #34191. --- be/src/cloud/cloud_internal_service.cpp | 1 + be/src/io/cache/block_file_cache_downloader.cpp | 2 +- gensrc/proto/internal_service.proto | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 63e3d4f30818fa..66e089c22e9d53 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -94,6 +94,7 @@ void CloudInternalServiceImpl::get_file_cache_meta_by_tablet_id( meta->set_rowset_id(rowset_id); meta->set_segment_id(segment_id); meta->set_file_name(file_name); + meta->set_file_size(rowset->rowset_meta()->segment_file_size(segment_id)); meta->set_offset(std::get<0>(tuple)); meta->set_size(std::get<1>(tuple)); meta->set_cache_type(cache_type_to_pb(std::get<2>(tuple))); diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index 02e8f736828cb1..6c73a2d59fa210 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -171,7 +171,7 @@ void FileCacheBlockDownloader::download_file_cache_block( DownloadFileMeta download_meta { .path = storage_resource.value()->remote_segment_path(*find_it->second, meta.segment_id()), - .file_size = meta.offset() + meta.size(), // To avoid trigger get file size IO + .file_size = meta.has_file_size() ? meta.file_size() : -1, // To avoid trigger get file size IO .offset = meta.offset(), .download_size = meta.size(), .file_system = storage_resource.value()->fs, diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 9997ecd478e56d..0637cc77b91d8f 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -808,6 +808,7 @@ message FileCacheBlockMeta { required int64 size = 6; required FileCacheType cache_type = 7; required int64 expiration_time = 8; + optional int64 file_size = 9; } message PGetFileCacheMetaResponse { From 2274dfd3c9d85fdcead6268bcb6f945beb35a76e Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Tue, 6 Aug 2024 14:51:47 +0800 Subject: [PATCH 2/3] fix --- be/src/olap/rowset/segment_v2/segment.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 1b31117f126aa2..035b3f3ccd7ce2 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -315,8 +315,8 @@ Status Segment::_parse_footer(SegmentFooterPB* footer) { DCHECK_EQ(bytes_read, 12); if (memcmp(fixed_buf + 8, k_segment_magic, k_segment_magic_length) != 0) { - return Status::Corruption("Bad segment file {}: magic number not match, cache_key: {}", - _file_reader->path().native(), + return Status::Corruption("Bad segment file {}: file_size: {}, magic number not match, cache_key: {}", + _file_reader->path().native(), file_size, file_cache_key_str(_file_reader->path().native())); } @@ -339,17 +339,18 @@ Status Segment::_parse_footer(SegmentFooterPB* footer) { uint32_t actual_checksum = crc32c::Value(footer_buf.data(), footer_buf.size()); if (actual_checksum != expect_checksum) { return Status::Corruption( - "Bad segment file {}: footer checksum not match, actual={} vs expect={}, " - "cache_key: {}", - _file_reader->path().native(), actual_checksum, expect_checksum, + "Bad segment file {}: file_size = {}, footer checksum not match, actual={} " + "vs expect={}, cache_key: {}", + _file_reader->path().native(), file_size, actual_checksum, expect_checksum, file_cache_key_str(_file_reader->path().native())); } // deserialize footer PB if (!footer->ParseFromString(footer_buf)) { return Status::Corruption( - "Bad segment file {}: failed to parse SegmentFooterPB, cache_key: ", - _file_reader->path().native(), file_cache_key_str(_file_reader->path().native())); + "Bad segment file {}: file_size = {}, failed to parse SegmentFooterPB, cache_key: ", + _file_reader->path().native(), file_size, + file_cache_key_str(_file_reader->path().native())); } return Status::OK(); } From 4d673208635540e2790fc74e002950977ec6baa8 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Tue, 6 Aug 2024 14:57:29 +0800 Subject: [PATCH 3/3] fix --- be/src/io/cache/block_file_cache_downloader.cpp | 3 ++- be/src/olap/rowset/segment_v2/segment.cpp | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index 6c73a2d59fa210..585c0ff015993b 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -171,7 +171,8 @@ void FileCacheBlockDownloader::download_file_cache_block( DownloadFileMeta download_meta { .path = storage_resource.value()->remote_segment_path(*find_it->second, meta.segment_id()), - .file_size = meta.has_file_size() ? meta.file_size() : -1, // To avoid trigger get file size IO + .file_size = meta.has_file_size() ? meta.file_size() + : -1, // To avoid trigger get file size IO .offset = meta.offset(), .download_size = meta.size(), .file_system = storage_resource.value()->fs, diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 035b3f3ccd7ce2..7f353cc0add199 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -315,9 +315,10 @@ Status Segment::_parse_footer(SegmentFooterPB* footer) { DCHECK_EQ(bytes_read, 12); if (memcmp(fixed_buf + 8, k_segment_magic, k_segment_magic_length) != 0) { - return Status::Corruption("Bad segment file {}: file_size: {}, magic number not match, cache_key: {}", - _file_reader->path().native(), file_size, - file_cache_key_str(_file_reader->path().native())); + return Status::Corruption( + "Bad segment file {}: file_size: {}, magic number not match, cache_key: {}", + _file_reader->path().native(), file_size, + file_cache_key_str(_file_reader->path().native())); } // read footer PB