From fcc5fc9ba088a6a606fb99659437a352c6f97a21 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 5 Mar 2025 10:45:33 +0800 Subject: [PATCH] [Fix](mow) Fix some logs for mow (#48523) 1. remove rows check for newly generated block in publish in partial update. When the rowset has multi segments, newly generated segments of conflict rows on these segments will be flushed concurrently using the same rowset writer. So `rowset_writer->num_rows()` may be larger than `new_generated_rows`. 2. correct `rowset_ids to add:` size when calculate delete bitmap in publish phase. Include invisible rowsets for txn_load 3. add a config `enable_mow_verbose_log` to control whether to print verbose log for delete bitmap calculation. --- be/src/common/config.cpp | 2 ++ be/src/common/config.h | 2 ++ be/src/olap/base_tablet.cpp | 12 ++++-------- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 48afc98cc05186..4ab55f0557b089 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1450,6 +1450,8 @@ DEFINE_mInt32(check_tablet_delete_bitmap_interval_seconds, "300"); DEFINE_mInt32(check_tablet_delete_bitmap_score_top_n, "10"); DEFINE_mBool(enable_check_tablet_delete_bitmap_score, "true"); +DEFINE_mBool(enable_mow_verbose_log, "false"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index bdb275f6dd9e6c..86733919d50ac0 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1523,6 +1523,8 @@ DECLARE_mInt32(check_tablet_delete_bitmap_interval_seconds); DECLARE_mInt32(check_tablet_delete_bitmap_score_top_n); DECLARE_mBool(enable_check_tablet_delete_bitmap_score); +DECLARE_mBool(enable_mow_verbose_log); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index 895899b9ca6cb5..a7e9126344c140 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -767,13 +767,8 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, rsid_to_rowset, &block)); RETURN_IF_ERROR(sort_block(block, ordered_block)); RETURN_IF_ERROR(rowset_writer->flush_single_block(&ordered_block)); - if (new_generated_rows != rowset_writer->num_rows()) { - LOG(WARNING) << "partial update correctness warning: conflict new generated rows (" - << new_generated_rows << ") not equal to the new flushed rows (" - << rowset_writer->num_rows() << "), tablet: " << tablet_id(); - } auto cost_us = watch.get_elapse_time_us(); - if (cost_us > 10 * 1000) { + if (config::enable_mow_verbose_log || cost_us > 10 * 1000) { LOG(INFO) << "calc segment delete bitmap for partial update, tablet: " << tablet_id() << " rowset: " << rowset_id << " seg_id: " << seg->id() << " dummy_version: " << end_version + 1 << " rows: " << seg->num_rows() @@ -785,7 +780,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, return Status::OK(); } auto cost_us = watch.get_elapse_time_us(); - if (cost_us > 10 * 1000) { + if (config::enable_mow_verbose_log || cost_us > 10 * 1000) { LOG(INFO) << "calc segment delete bitmap, tablet: " << tablet_id() << " rowset: " << rowset_id << " seg_id: " << seg->id() << " dummy_version: " << end_version + 1 << " rows: " << seg->num_rows() @@ -1352,7 +1347,8 @@ Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, TabletTxnInf RETURN_IF_ERROR(self->save_delete_bitmap(txn_info, txn_id, delete_bitmap, transient_rs_writer.get(), cur_rowset_ids)); LOG(INFO) << "[Publish] construct delete bitmap tablet: " << self->tablet_id() - << ", rowset_ids to add: " << rowset_ids_to_add.size() + << ", rowset_ids to add: " + << (specified_rowsets.size() + rowsets_skip_alignment.size()) << ", rowset_ids to del: " << rowset_ids_to_del.size() << ", cur version: " << cur_version << ", transaction_id: " << txn_id << "," << ss.str() << " , total rows: " << total_rows