diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index bc8b13b8a5a9ae..eb6450539fe743 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1455,6 +1455,8 @@ DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true"); DEFINE_mBool(enable_mow_verbose_log, "false"); +DEFINE_mBool(skip_version_col_when_calc_check_sum, "false"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index 8105e9fc48c7d6..6b1690d7f9f3af 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1530,6 +1530,8 @@ DECLARE_mBool(enable_prune_delete_sign_when_base_compaction); DECLARE_mBool(enable_mow_verbose_log); +DECLARE_mBool(skip_version_col_when_calc_check_sum); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index 8c20d542c962d0..5dd869cea3da28 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -52,6 +52,7 @@ #include "olap/tablet_reader.h" #include "olap/utils.h" #include "util/slice.h" +#include "vec/columns/columns_number.h" #include "vec/core/block.h" #include "vec/olap/block_reader.h" #include "vec/olap/vertical_block_reader.h" @@ -120,6 +121,25 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, RETURN_NOT_OK_STATUS_WITH_WARN(reader.next_block_with_aggregation(&block, &eof), "failed to read next block when merging rowsets of tablet " + std::to_string(tablet->tablet_id())); +#ifndef NDEBUG + if (auto* version_col = block.try_get_by_name(VERSION_COL); version_col != nullptr) { + auto* col = assert_cast( + version_col->column->assume_mutable().get()); + int64_t max_version = tablet->max_version_unlocked(); + for (auto val : col->get_data()) { + if (val > max_version) { + auto msg = fmt::format( + "[Merger::vmerge_rowsets] tablet={}, tablet's max_version={}, but " + "found value={} in __DORIS_VERSION_COL__ when merge rowsets for " + "version={}, reader_type={}", + tablet->tablet_id(), max_version, val, + reader_params.version.to_string(), reader_type); + LOG_WARNING(msg); + DCHECK(false) << msg; + } + } + } +#endif RETURN_NOT_OK_STATUS_WITH_WARN(dst_rowset_writer->add_block(&block), "failed to write block when merging rowsets of tablet " + std::to_string(tablet->tablet_id())); @@ -295,6 +315,25 @@ Status Merger::vertical_compact_one_group( RETURN_NOT_OK_STATUS_WITH_WARN(reader.next_block_with_aggregation(&block, &eof), "failed to read next block when merging rowsets of tablet " + std::to_string(tablet->tablet_id())); +#ifndef NDEBUG + if (auto* version_col = block.try_get_by_name(VERSION_COL); version_col != nullptr) { + auto* col = assert_cast( + version_col->column->assume_mutable().get()); + int64_t max_version = tablet->max_version_unlocked(); + for (auto val : col->get_data()) { + if (val > max_version) { + auto msg = fmt::format( + "[Merger::vertical_compact_one_group] tablet={}, tablet's " + "max_version={}, but found value={} in __DORIS_VERSION_COL__ when " + "merge rowsets for version={}, reader_type={}", + tablet->tablet_id(), max_version, val, + reader_params.version.to_string(), reader_type); + LOG_WARNING(msg); + DCHECK(false) << msg; + } + } + } +#endif RETURN_NOT_OK_STATUS_WITH_WARN( dst_rowset_writer->add_columns(&block, column_group, is_key, max_rows_per_segment, has_cluster_key), diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index 94e4ee75f36761..28534b8b4184a0 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -822,8 +822,12 @@ void Block::shuffle_columns(const std::vector& result_column_ids) { } void Block::update_hash(SipHash& hash) const { + bool skip_version_col = config::skip_version_col_when_calc_check_sum; for (size_t row_no = 0, num_rows = rows(); row_no < num_rows; ++row_no) { for (const auto& col : data) { + if (skip_version_col && col.name == VERSION_COL) { + continue; + } col.column->update_hash_with_value(row_no, hash); } }