From e145ce740f0bb57de8d3e3ec246e39aa45336f3c Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Tue, 11 Feb 2025 11:03:24 +0800 Subject: [PATCH] [log](mow) reduce log for data load on mow table (apache#47647) --- .../cloud/cloud_txn_delete_bitmap_cache.cpp | 5 ---- be/src/olap/base_tablet.cpp | 30 +++++++++++-------- be/src/olap/rowset_builder.cpp | 2 -- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp b/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp index 55f76316fb7f26..95077f5c8884c6 100644 --- a/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp +++ b/be/src/cloud/cloud_txn_delete_bitmap_cache.cpp @@ -202,11 +202,6 @@ Status CloudTxnDeleteBitmapCache::update_tablet_txn_info(TTransactionId transact // must call release handle to reduce the reference count, // otherwise there will be memory leak release(handle); - LOG_INFO("update txn related delete bitmap") - .tag("txn_id", transaction_id) - .tag("tablt_id", tablet_id) - .tag("delete_bitmap_size", charge) - .tag("publish_status", static_cast(publish_status)); return Status::OK(); } diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index ae921a76c617d1..ff01a898a31543 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -536,10 +536,7 @@ Status BaseTablet::calc_delete_bitmap(const BaseTabletSPtr& tablet, RowsetShared const std::vector& specified_rowsets, DeleteBitmapPtr delete_bitmap, int64_t end_version, CalcDeleteBitmapToken* token, RowsetWriter* rowset_writer) { - auto rowset_id = rowset->rowset_id(); if (specified_rowsets.empty() || segments.empty()) { - LOG(INFO) << "skip to construct delete bitmap tablet: " << tablet->tablet_id() - << " rowset: " << rowset_id; return Status::OK(); } @@ -759,20 +756,27 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, << new_generated_rows << ") not equal to the new flushed rows (" << rowset_writer->num_rows() << "), tablet: " << tablet_id(); } - LOG(INFO) << "calc segment delete bitmap for partial update, tablet: " << tablet_id() + auto cost_us = watch.get_elapse_time_us(); + if (cost_us > 10 * 1000) { + LOG(INFO) << "calc segment delete bitmap for partial update, tablet: " << tablet_id() + << " rowset: " << rowset_id << " seg_id: " << seg->id() + << " dummy_version: " << end_version + 1 << " rows: " << seg->num_rows() + << " conflict rows: " << conflict_rows + << " new generated rows: " << new_generated_rows + << " bimap num: " << delete_bitmap->delete_bitmap.size() + << " cost: " << cost_us << "(us)"; + } + return Status::OK(); + } + auto cost_us = watch.get_elapse_time_us(); + if (cost_us > 10 * 1000) { + LOG(INFO) << "calc segment delete bitmap, tablet: " << tablet_id() << " rowset: " << rowset_id << " seg_id: " << seg->id() << " dummy_version: " << end_version + 1 << " rows: " << seg->num_rows() << " conflict rows: " << conflict_rows - << " new generated rows: " << new_generated_rows - << " bimap num: " << delete_bitmap->delete_bitmap.size() - << " cost: " << watch.get_elapse_time_us() << "(us)"; - return Status::OK(); + << " bitmap num: " << delete_bitmap->delete_bitmap.size() << " cost: " << cost_us + << "(us)"; } - LOG(INFO) << "calc segment delete bitmap, tablet: " << tablet_id() << " rowset: " << rowset_id - << " seg_id: " << seg->id() << " dummy_version: " << end_version + 1 - << " rows: " << seg->num_rows() << " conflict rows: " << conflict_rows - << " bitmap num: " << delete_bitmap->delete_bitmap.size() - << " cost: " << watch.get_elapse_time_us() << "(us)"; return Status::OK(); } diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index 63ffe4f68a1f86..192118832f436e 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -311,8 +311,6 @@ Status BaseRowsetBuilder::wait_calc_delete_bitmap() { std::lock_guard l(_lock); SCOPED_TIMER(_wait_delete_bitmap_timer); RETURN_IF_ERROR(_calc_delete_bitmap_token->wait()); - LOG(INFO) << "Got result of calc delete bitmap task from executor, tablet_id: " - << _tablet->tablet_id() << ", txn_id: " << _req.txn_id; return Status::OK(); }