From d62efcb2958e91ef5cf07a52dc74045653b0facd Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Wed, 18 Sep 2024 17:59:34 +0800 Subject: [PATCH] [fix](compaction) catch exception in compaction (#40900) ## Proposed changes ``` terminate called after throwing an instance of 'doris::Exception' {color:red} what(): [E6] Too large string size.{color} 0# doris::Exception::Exception(int, std::basic_string_view > const&) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:173 1# doris::vectorized::read_string_binary(std::__cxx11::basic_string, std::allocator >&, doris::vectorized::BufferReadable&, unsigned long) at /root/doris_branch-2.1/doris/be/src/vec/io/io_helper.h:177 2# doris::vectorized::IAggregateFunctionHelper, true> >::deserialize_and_merge_from_column_range(char*, doris::vectorized::IColumn const&, unsigned long, unsigned long, doris::vectorized::Arena*) const at /root/doris_branch-2.1/doris/be/src/vec/aggregate_functions/aggregate_function.h:0 3# doris::vectorized::IAggregateFunctionHelper::add_batch_range(unsigned long, unsigned long, char*, doris::vectorized::IColumn const**, doris::vectorized::Arena*, bool) at /root/doris_branch-2.1/doris/be/src/vec/aggregate_functions/aggregate_function.h:0 4# doris::vectorized::VerticalBlockReader::_update_agg_value(std::vector::mutable_ptr, std::allocator::mutable_ptr > >&, int, int, bool) at /root/doris_branch-2.1/doris/be/src/vec/olap/vertical_block_reader.cpp:326 5# doris::vectorized::VerticalBlockReader::_update_agg_data(std::vector::mutable_ptr, std::allocator::mutable_ptr > >&) at /root/doris_branch-2.1/doris/be/src/vec/olap/vertical_block_reader.cpp:308 6# doris::vectorized::VerticalBlockReader::_agg_key_next_block(doris::vectorized::Block*, bool*) at /root/doris_branch-2.1/doris/be/src/vec/olap/vertical_block_reader.cpp:0 7# doris::vectorized::VerticalBlockReader::next_block_with_aggregation(doris::vectorized::Block*, bool*) at /root/doris_branch-2.1/doris/be/src/common/status.h:491 8# doris::Merger::vertical_compact_one_group(std::shared_ptr, doris::ReaderType, std::shared_ptr, bool, std::vector > const&, doris::vectorized::RowSourcesBuffer*, std::vector, std::allocator > > const&, doris::RowsetWriter*, long, doris::Merger::Statistics*, std::vector >, long, doris::CompactionSampleInfo*) at /root/doris_branch-2.1/doris/be/src/olap/merger.cpp:0 9# doris::Merger::vertical_merge_rowsets(std::shared_ptr, doris::ReaderType, std::shared_ptr, std::vector, std::allocator > > const&, doris::RowsetWriter*, long, long, doris::Merger::Statistics*) at /root/doris_branch-2.1/doris/be/src/olap/merger.cpp:445 10# doris::Compaction::do_compaction_impl(long) at /root/doris_branch-2.1/doris/be/src/olap/compaction.cpp:385 11# doris::Compaction::do_compaction(long) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/shared_ptr_base.h:1291 12# doris::CumulativeCompaction::execute_compact_impl() at /root/doris_branch-2.1/doris/be/src/common/status.h:491 13# doris::Compaction::execute_compact() at /root/doris_branch-2.1/doris/be/src/common/status.h:491 14# doris::Tablet::execute_compaction(doris::Compaction&) at /root/doris_branch-2.1/doris/be/src/common/status.h:491 15# std::_Function_handler, doris::CompactionType, bool)::$_0>::_M_invoke(std::_Any_data const&) at /root/doris_branch-2.1/doris/be/src/olap/olap_server.cpp:1018 16# doris::ThreadPool::dispatch_thread() at /root/doris_branch-2.1/doris/be/src/util/threadpool.cpp:0 17# doris::Thread::supervise_thread(void*) at /var/local/ldb-toolchain/bin/../usr/include/pthread.h:562 18# ? 19# ? *** Query id: 0-0 *** *** is nereids: 0 *** *** tablet id: 38266341 *** *** Aborted at 1726615633 (unix time) try "date -d @1726615633" if you are using GNU date *** *** Current BE git commitID: db06c678a3 *** *** SIGABRT unknown detail explain (@0x26af02) received by PID 2535170 (TID 2536168 OR 0x7f48c81cd640) from PID 2535170; stack trace: *** 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /root/doris_branch-2.1/doris/be/src/common/signal_handler.h:421 1# 0x00007F4AB18F8520 in /lib/x86_64-linux-gnu/libc.so.6 2# pthread_kill at ./nptl/pthread_kill.c:89 3# raise at ../sysdeps/posix/raise.c:27 4# abort at ./stdlib/abort.c:81 5# __gnu_cxx::__verbose_terminate_handler() [clone .cold] at ../../../../libstdc++-v3/libsupc++/vterminate.cc:75 6# __cxxabiv1::__terminate(void (*)()) at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:48 7# 0x000055C401EF7811 in /mnt/disk1/STRESS_ENV/be/lib/doris_be 8# 0x000055C401EF7964 in /mnt/disk1/STRESS_ENV/be/lib/doris_be 9# doris::vectorized::read_string_binary(std::__cxx11::basic_string, std::allocator >&, doris::vectorized::BufferReadable&, unsigned long) at /root/doris_branch-2.1/doris/be/src/vec/io/io_helper.h:180 10# doris::vectorized::IAggregateFunctionHelper, true> >::deserialize_and_merge_from_column_range(char*, doris::vectorized::IColumn const&, unsigned long, unsigned long, doris::vectorized::Arena*) const in /mnt/disk1/STRESS_ENV/be/lib/doris_be 11# doris::vectorized::IAggregateFunctionHelper::add_batch_range(unsigned long, unsigned long, char*, doris::vectorized::IColumn const**, doris::vectorized::Arena*, bool) in /mnt/disk1/STRESS_ENV/be/lib/doris_be 12# doris::vectorized::VerticalBlockReader::_update_agg_value(std::vector::mutable_ptr, std::allocator::mutable_ptr > >&, int, int, bool) at /root/doris_branch-2.1/doris/be/src/vec/olap/vertical_block_reader.cpp:326 13# doris::vectorized::VerticalBlockReader::_update_agg_data(std::vector::mutable_ptr, std::allocator::mutable_ptr > >&) at /root/doris_branch-2.1/doris/be/src/vec/olap/vertical_block_reader.cpp:308 14# doris::vectorized::VerticalBlockReader::_agg_key_next_block(doris::vectorized::Block*, bool*) in /mnt/disk1/STRESS_ENV/be/lib/doris_be 15# doris::vectorized::VerticalBlockReader::next_block_with_aggregation(doris::vectorized::Block*, bool*) at /root/doris_branch-2.1/doris/be/src/vec/olap/vertical_block_reader.cpp:58 16# doris::Merger::vertical_compact_one_group(std::shared_ptr, doris::ReaderType, std::shared_ptr, bool, std::vector > const&, doris::vectorized::RowSourcesBuffer*, std::vector, std::allocator > > const&, doris::RowsetWriter*, long, doris::Merger::Statistics*, std::vector >, long, doris::CompactionSampleInfo*) in /mnt/disk1/STRESS_ENV/be/lib/doris_be 17# doris::Merger::vertical_merge_rowsets(std::shared_ptr, doris::ReaderType, std::shared_ptr, std::vector, std::allocator > > const&, doris::RowsetWriter*, long, long, doris::Merger::Statistics*) at /root/doris_branch-2.1/doris/be/src/olap/merger.cpp:445 18# doris::Compaction::do_compaction_impl(long) at /root/doris_branch-2.1/doris/be/src/olap/compaction.cpp:385 19# doris::Compaction::do_compaction(long) at /root/doris_branch-2.1/doris/be/src/olap/compaction.cpp:136 20# doris::CumulativeCompaction::execute_compact_impl() at /root/doris_branch-2.1/doris/be/src/olap/cumulative_compaction.cpp:79 21# doris::Compaction::execute_compact() at /root/doris_branch-2.1/doris/be/src/olap/compaction.cpp:118 22# doris::Tablet::execute_compaction(doris::Compaction&) at /root/doris_branch-2.1/doris/be/src/olap/tablet.cpp:2067 23# std::_Function_handler, doris::CompactionType, bool)::$_0>::_M_invoke(std::_Any_data const&) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291 24# doris::ThreadPool::dispatch_thread() in /mnt/disk1/STRESS_ENV/be/lib/doris_be 25# doris::Thread::supervise_thread(void*) at /root/doris_branch-2.1/doris/be/src/util/thread.cpp:499 26# start_thread at ./nptl/pthread_create.c:442 27# 0x00007F4AB19DC850 at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:83 ``` --- be/src/olap/tablet.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index d5d12c101f5657..dede9f2600f5a5 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -57,6 +57,7 @@ #include "agent/utils.h" #include "common/config.h" #include "common/consts.h" +#include "common/exception.h" #include "common/logging.h" #include "common/signal_handler.h" #include "common/status.h" @@ -2059,8 +2060,7 @@ void Tablet::execute_compaction(Compaction& compaction) { MonotonicStopWatch watch; watch.start(); - Status res = compaction.execute_compact(); - + Status res = [&]() { RETURN_IF_CATCH_EXCEPTION({ return compaction.execute_compact(); }); }(); if (!res.ok()) [[unlikely]] { set_last_failure_time(this, compaction, UnixMillis()); LOG(WARNING) << "failed to do " << compaction.compaction_name()