From e197079523c0a927dd0f6db5be529d3b52276d24 Mon Sep 17 00:00:00 2001 From: hui lai <1353307710@qq.com> Date: Sun, 20 Oct 2024 10:40:37 +0800 Subject: [PATCH] [fix](routine load) reduce routine load task consume log (#42058) There is too much routine load task log, five million logs were generated in 10 minutes. ``` grep 'consumer meet partition eof' be.INFO.log.20240930-164533 | wc -l 5369624 ``` --- be/src/runtime/routine_load/data_consumer.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/be/src/runtime/routine_load/data_consumer.cpp b/be/src/runtime/routine_load/data_consumer.cpp index 17ceee5d4c8d89..c6de4849d32a36 100644 --- a/be/src/runtime/routine_load/data_consumer.cpp +++ b/be/src/runtime/routine_load/data_consumer.cpp @@ -261,12 +261,13 @@ Status KafkaDataConsumer::group_consume(BlockingQueue* queue, } [[fallthrough]]; case RdKafka::ERR__PARTITION_EOF: { - LOG(INFO) << "consumer meet partition eof: " << _id - << " partition offset: " << msg->offset(); + VLOG_NOTICE << "consumer meet partition eof: " << _id + << " partition offset: " << msg->offset(); _consuming_partition_ids.erase(msg->partition()); if (!queue->blocking_put(msg.get())) { done = true; } else if (_consuming_partition_ids.size() <= 0) { + LOG(INFO) << "all partitions meet eof: " << _id; msg.release(); done = true; } else {