[Bug](array-type) Fix memory buffer overflow #13074

xy720 · 2022-09-29T16:51:02Z

Proposed changes

How to reproduce?

mysql> create table tbl (
    id int, 
    c_array array<int(11)>
) 
duplicate key(id) 
distributed by hash(id) buckets 1 
properties(
    'replication_num' = '2'
);

mysql> insert into tbl select k1, collect_list(k3) from test_query_qa.test group by k1;
Query OK, 255 rows affected (0.34 sec)
{'label':'insert_30b676178e12467d_8225f747fcbe3104', 'status':'VISIBLE', 'txnId':'1014'}

mysql> select c_array from tbl;

Be will core.
This bug is caused by a memory buffer overflow.

Coredump:

*** SIGSEGV address not mapped to object (@0x7ff28588e000) received by PID 80401 (TID 0x7ff2fb3d7700) from PID 18446744071654924288; stack trace: ***
 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/common/signal_handler.h:420
 1# 0x00007FF342660570 in /lib64/libc.so.6
 2# __GI_memcpy in /lib64/libc.so.6
 3# __asan_memcpy in /home/disk4/xuyang/work/be/lib/palo_be
 4# doris::vectorized::ColumnVector<int>::insert_many_in_copy_way(char const*, unsigned long) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/columns/column_vector.h:175
 5# doris::vectorized::ColumnVector<int>::insert_many_fix_len_data(char const*, unsigned long) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/columns/column_vector.h:218
 6# doris::vectorized::ColumnNullable::insert_many_fix_len_data(char const*, unsigned long) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/columns/column_nullable.h:112
 7# doris::segment_v2::BitShufflePageDecoder<(doris::FieldType)5>::next_batch(unsigned long*, COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn>&) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/olap/rowset/segment_v2/bitshuffle_page.h:402
 8# doris::segment_v2::FileColumnIterator::next_batch(unsigned long*, COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn>&, bool*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/olap/rowset/segment_v2/column_reader.cpp:827
 9# doris::segment_v2::ArrayFileColumnIterator::next_batch(unsigned long*, COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn>&, bool*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/olap/rowset/segment_v2/column_reader.cpp:602
10# doris::segment_v2::ColumnIterator::next_batch(unsigned long*, COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn>&) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/olap/rowset/segment_v2/column_reader.h:249
11# doris::segment_v2::SegmentIterator::_read_columns(std::vector<unsigned int, std::allocator<unsigned int> > const&, std::vector<COW<doris::vectorized::IColumn>::mutable_ptr<doris::vectorized::IColumn>, std::allocator<COW<doris::vectorized::IColumn>::mutable_ptr<doris::
vectorized::IColumn> > >&, unsigned long) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/olap/rowset/segment_v2/segment_iterator.cpp:908
12# doris::segment_v2::SegmentIterator::_read_columns_by_index(unsigned int, unsigned int&, bool) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/olap/rowset/segment_v2/segment_iterator.cpp:995
13# doris::segment_v2::SegmentIterator::next_batch(doris::vectorized::Block*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/olap/rowset/segment_v2/segment_iterator.cpp:1153
14# doris::BetaRowsetReader::next_block(doris::vectorized::Block*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/olap/rowset/beta_rowset_reader.cpp:276
15# doris::vectorized::VCollectIterator::Level0Iterator::next(doris::vectorized::Block*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/olap/vcollect_iterator.cpp:261
16# doris::vectorized::VCollectIterator::Level1Iterator::_normal_next(doris::vectorized::Block*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/olap/vcollect_iterator.cpp:527
17# doris::vectorized::VCollectIterator::Level1Iterator::_normal_next(doris::vectorized::Block*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/olap/vcollect_iterator.cpp:536
18# doris::vectorized::VCollectIterator::Level1Iterator::next(doris::vectorized::Block*) in /home/disk4/xuyang/work/be/lib/palo_be
19# doris::vectorized::VCollectIterator::next(doris::vectorized::Block*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/olap/vcollect_iterator.cpp:181
20# doris::vectorized::BlockReader::_direct_next_block(doris::vectorized::Block*, doris::MemPool*, doris::ObjectPool*, bool*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/olap/block_reader.cpp:174
21# doris::vectorized::BlockReader::next_block_with_aggregation(doris::vectorized::Block*, doris::MemPool*, doris::ObjectPool*, bool*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/olap/block_reader.h:45
22# doris::vectorized::NewOlapScanner::_get_block_impl(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/exec/scan/new_olap_scanner.cpp:308
23# doris::vectorized::VScanner::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/exec/scan/vscanner.cpp:55
24# doris::vectorized::ScannerScheduler::_scanner_scan(doris::vectorized::ScannerScheduler*, doris::vectorized::ScannerContext*, doris::vectorized::VScanner*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/exec/scan/scanner_scheduler.cpp:243
25# doris::vectorized::ScannerScheduler::_schedule_scanners(doris::vectorized::ScannerContext*)::$_1::operator()() const at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/vec/exec/scan/scanner_scheduler.cpp:146
26# void std::__invoke_impl<void, doris::vectorized::ScannerScheduler::_schedule_scanners(doris::vectorized::ScannerContext*)::$_1&>(std::__invoke_other, doris::vectorized::ScannerScheduler::_schedule_scanners(doris::vectorized::ScannerContext*)::$_1&) at /home/disk4/xuya
ng/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61
27# std::enable_if<is_invocable_r_v<void, doris::vectorized::ScannerScheduler::_schedule_scanners(doris::vectorized::ScannerContext*)::$_1&>, void>::type std::__invoke_r<void, doris::vectorized::ScannerScheduler::_schedule_scanners(doris::vectorized::ScannerContext*)::$_1
&>(doris::vectorized::ScannerScheduler::_schedule_scanners(doris::vectorized::ScannerContext*)::$_1&) at /home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:117
28# std::_Function_handler<void (), doris::vectorized::ScannerScheduler::_schedule_scanners(doris::vectorized::ScannerContext*)::$_1>::_M_invoke(std::_Any_data const&) at /home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/
11/../../../../include/c++/11/bits/std_function.h:291
29# std::function<void ()>::operator()() const at /home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:560
30# doris::FunctionRunnable::run() at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/util/threadpool.cpp:45
31# doris::ThreadPool::dispatch_thread() at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/util/threadpool.cpp:540
32# void std::__invoke_impl<void, void (doris::ThreadPool::*&)(), doris::ThreadPool*&>(std::__invoke_memfun_deref, void (doris::ThreadPool::*&)(), doris::ThreadPool*&) at /home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/
11/../../../../include/c++/11/bits/invoke.h:74
33# std::__invoke_result<void (doris::ThreadPool::*&)(), doris::ThreadPool*&>::type std::__invoke<void (doris::ThreadPool::*&)(), doris::ThreadPool*&>(void (doris::ThreadPool::*&)(), doris::ThreadPool*&) at /home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolch
ain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96
34# void std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>::__call<void, , 0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) at /home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/functi
onal:420
35# void std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>::operator()<, void>() at /home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/functional:503
36# void std::__invoke_impl<void, std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>&>(std::__invoke_other, std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>&) at /home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x8
6_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61
37# std::enable_if<is_invocable_r_v<void, std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>&>, void>::type std::__invoke_r<void, std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>&>(std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()>&) at
/home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:117
38# std::_Function_handler<void (), std::_Bind<void (doris::ThreadPool::*(doris::ThreadPool*))()> >::_M_invoke(std::_Any_data const&) at /home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits
/std_function.h:291
39# std::function<void ()>::operator()() const at /home/disk4/xuyang/work/baidu/bdg/doris/palo-toolchain/ldb_toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:560
40# doris::Thread::supervise_thread(void*) at /home/disk4/xuyang/work/baidu/bdg/doris/core/be/src/util/thread.cpp:425
41# start_thread in /lib64/libpthread.so.0
42# __clone in /lib64/libc.so.6

Problem summary

Describe your changes.

Checklist(Required)

Does it affect the original behavior:
- Yes
- No
- I don't know
Has unit tests been added:
- Yes
- No
- No Need
Has document been added or modified:
- Yes
- No
- No Need
Does it need to update dependencies:
- Yes
- No
Are there any changes that cannot be rolled back:
- Yes (If Yes, please explain WHY)
- No

Further comments

If this is a relatively large or complex change, kick off the discussion at dev@doris.apache.org by explaining why you chose the solution you did and what alternatives you considered, etc...

be/src/olap/rowset/segment_v2/column_reader.cpp

dataroaring · 2022-10-07T08:10:48Z

please add a test case?

xy720 · 2022-10-08T13:08:00Z

Add a test case.

cambyzju

LGTM

github-actions · 2022-10-08T13:47:53Z

PR approved by at least one committer and no changes requested.

github-actions · 2022-10-08T13:47:55Z

PR approved by anyone and no changes requested.

cambyzju

LGTM

github-actions · 2022-10-10T03:41:30Z

PR approved by at least one committer and no changes requested.

As mentioned in #13074, there will be some problem in ColumnVector<int>::insert_many_in_copy_way. Column::insert_xxx functions will append some data, they should reserve or resize before append data. Co-authored-by: cambyzju <zhuxiaoli01@baidu.com>

xy720 requested a review from cambyzju September 29, 2022 16:54

cambyzju reviewed Sep 30, 2022

View reviewed changes

be/src/olap/rowset/segment_v2/column_reader.cpp Outdated Show resolved Hide resolved

cambyzju mentioned this pull request Sep 30, 2022

[refractor](vectorized) refractor some insert_xxx functions #13088

Merged

13 tasks

xy720 force-pushed the fix-segment-fault-when-select-array branch 2 times, most recently from f957842 to e84a157 Compare October 8, 2022 12:48

github-actions bot added the kind/test label Oct 8, 2022

cambyzju previously approved these changes Oct 8, 2022

View reviewed changes

github-actions bot added the approved Indicates a PR has been approved by one committer. label Oct 8, 2022

github-actions bot added the reviewed label Oct 8, 2022

xy720 dismissed cambyzju’s stale review via f41451b October 8, 2022 14:04

xy720 force-pushed the fix-segment-fault-when-select-array branch from cbe5be2 to f41451b Compare October 8, 2022 14:04

github-actions bot removed the approved Indicates a PR has been approved by one committer. label Oct 8, 2022

xy720 added 4 commits October 9, 2022 11:45

save code

4fc28e5

save

2153315

save

b7a8f7b

save

8dd83c0

xy720 force-pushed the fix-segment-fault-when-select-array branch from f41451b to 8dd83c0 Compare October 9, 2022 03:46

cambyzju approved these changes Oct 10, 2022

View reviewed changes

github-actions bot added the approved Indicates a PR has been approved by one committer. label Oct 10, 2022

cambyzju merged commit 20b583c into apache:master Oct 10, 2022

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[Bug](array-type) Fix memory buffer overflow #13074

[Bug](array-type) Fix memory buffer overflow #13074

Uh oh!

xy720 commented Sep 29, 2022 •

edited

Loading

Uh oh!

Uh oh!

dataroaring commented Oct 7, 2022

Uh oh!

xy720 commented Oct 8, 2022

Uh oh!

cambyzju left a comment

Uh oh!

github-actions bot commented Oct 8, 2022

Uh oh!

github-actions bot commented Oct 8, 2022

Uh oh!

cambyzju left a comment

Uh oh!

github-actions bot commented Oct 10, 2022

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

[Bug](array-type) Fix memory buffer overflow #13074

[Bug](array-type) Fix memory buffer overflow #13074

Uh oh!

Conversation

xy720 commented Sep 29, 2022 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Proposed changes

Problem summary

Checklist(Required)

Further comments

Uh oh!

Uh oh!

dataroaring commented Oct 7, 2022

Uh oh!

xy720 commented Oct 8, 2022

Uh oh!

cambyzju left a comment

Choose a reason for hiding this comment

Uh oh!

github-actions bot commented Oct 8, 2022

Uh oh!

github-actions bot commented Oct 8, 2022

Uh oh!

cambyzju left a comment

Choose a reason for hiding this comment

Uh oh!

github-actions bot commented Oct 10, 2022

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

xy720 commented Sep 29, 2022 •

edited

Loading