From 832aeb26c2a23dde5ba602aab6c9174bd3ca6e90 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 17 Mar 2025 16:09:55 +0800 Subject: [PATCH 1/3] update --- be/src/common/config.cpp | 2 ++ be/src/common/config.h | 2 ++ be/src/vec/core/block.cpp | 9 +++++++++ 3 files changed, 13 insertions(+) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index bc8b13b8a5a9ae..eb6450539fe743 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1455,6 +1455,8 @@ DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true"); DEFINE_mBool(enable_mow_verbose_log, "false"); +DEFINE_mBool(skip_version_col_when_calc_check_sum, "false"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 8105e9fc48c7d6..6b1690d7f9f3af 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1530,6 +1530,8 @@ DECLARE_mBool(enable_prune_delete_sign_when_base_compaction); DECLARE_mBool(enable_mow_verbose_log); +DECLARE_mBool(skip_version_col_when_calc_check_sum); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index 94e4ee75f36761..11f1fbfb5440d9 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -822,8 +822,17 @@ void Block::shuffle_columns(const std::vector& result_column_ids) { } void Block::update_hash(SipHash& hash) const { + bool skip_version_col = config::skip_version_col_when_calc_check_sum; + std::string msg {}; + for (const auto& name : get_names()) { + msg += fmt::format("{},", name); + } + LOG_INFO("[xxx Block::update_hash] {}", msg); for (size_t row_no = 0, num_rows = rows(); row_no < num_rows; ++row_no) { for (const auto& col : data) { + if (skip_version_col && col.name == VERSION_COL) { + continue; + } col.column->update_hash_with_value(row_no, hash); } } From 5ac35ce2ff4b325c84d1c0822449af39e3f58c09 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 17 Mar 2025 17:24:25 +0800 Subject: [PATCH 2/3] remove log --- be/src/vec/core/block.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index 11f1fbfb5440d9..28534b8b4184a0 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -823,11 +823,6 @@ void Block::shuffle_columns(const std::vector& result_column_ids) { void Block::update_hash(SipHash& hash) const { bool skip_version_col = config::skip_version_col_when_calc_check_sum; - std::string msg {}; - for (const auto& name : get_names()) { - msg += fmt::format("{},", name); - } - LOG_INFO("[xxx Block::update_hash] {}", msg); for (size_t row_no = 0, num_rows = rows(); row_no < num_rows; ++row_no) { for (const auto& col : data) { if (skip_version_col && col.name == VERSION_COL) { From a141d5c2cbce772e59fea433b68e1e1c5caeb003 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 18 Mar 2025 10:37:44 +0800 Subject: [PATCH 3/3] add dcheck --- be/src/olap/merger.cpp | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index 8c20d542c962d0..5dd869cea3da28 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -52,6 +52,7 @@ #include "olap/tablet_reader.h" #include "olap/utils.h" #include "util/slice.h" +#include "vec/columns/columns_number.h" #include "vec/core/block.h" #include "vec/olap/block_reader.h" #include "vec/olap/vertical_block_reader.h" @@ -120,6 +121,25 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, RETURN_NOT_OK_STATUS_WITH_WARN(reader.next_block_with_aggregation(&block, &eof), "failed to read next block when merging rowsets of tablet " + std::to_string(tablet->tablet_id())); +#ifndef NDEBUG + if (auto* version_col = block.try_get_by_name(VERSION_COL); version_col != nullptr) { + auto* col = assert_cast( + version_col->column->assume_mutable().get()); + int64_t max_version = tablet->max_version_unlocked(); + for (auto val : col->get_data()) { + if (val > max_version) { + auto msg = fmt::format( + "[Merger::vmerge_rowsets] tablet={}, tablet's max_version={}, but " + "found value={} in __DORIS_VERSION_COL__ when merge rowsets for " + "version={}, reader_type={}", + tablet->tablet_id(), max_version, val, + reader_params.version.to_string(), reader_type); + LOG_WARNING(msg); + DCHECK(false) << msg; + } + } + } +#endif RETURN_NOT_OK_STATUS_WITH_WARN(dst_rowset_writer->add_block(&block), "failed to write block when merging rowsets of tablet " + std::to_string(tablet->tablet_id())); @@ -295,6 +315,25 @@ Status Merger::vertical_compact_one_group( RETURN_NOT_OK_STATUS_WITH_WARN(reader.next_block_with_aggregation(&block, &eof), "failed to read next block when merging rowsets of tablet " + std::to_string(tablet->tablet_id())); +#ifndef NDEBUG + if (auto* version_col = block.try_get_by_name(VERSION_COL); version_col != nullptr) { + auto* col = assert_cast( + version_col->column->assume_mutable().get()); + int64_t max_version = tablet->max_version_unlocked(); + for (auto val : col->get_data()) { + if (val > max_version) { + auto msg = fmt::format( + "[Merger::vertical_compact_one_group] tablet={}, tablet's " + "max_version={}, but found value={} in __DORIS_VERSION_COL__ when " + "merge rowsets for version={}, reader_type={}", + tablet->tablet_id(), max_version, val, + reader_params.version.to_string(), reader_type); + LOG_WARNING(msg); + DCHECK(false) << msg; + } + } + } +#endif RETURN_NOT_OK_STATUS_WITH_WARN( dst_rowset_writer->add_columns(&block, column_group, is_key, max_rows_per_segment, has_cluster_key),