From 99ff0bb7cea6b04375e22b4e96e52ffb4e475119 Mon Sep 17 00:00:00 2001 From: nextdreamblue Date: Fri, 9 Dec 2022 20:03:48 +0800 Subject: [PATCH 1/6] debug --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 6ed3c5ac6f6282..c977132bb39424 100755 --- a/build.sh +++ b/build.sh @@ -83,7 +83,7 @@ clean_be() { # "build.sh --clean" just cleans and exits, however CMAKE_BUILD_DIR is set # while building be. - CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" + CMAKE_BUILD_TYPE="${BUILD_TYPE:-DEBUG}" CMAKE_BUILD_DIR="${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE}" rm -rf "${CMAKE_BUILD_DIR}" @@ -387,7 +387,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then if [[ -e "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" ]]; then rm -f "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" fi - CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" + CMAKE_BUILD_TYPE="${BUILD_TYPE:-DEBUG}" echo "Build Backend: ${CMAKE_BUILD_TYPE}" CMAKE_BUILD_DIR="${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE}" if [[ "${CLEAN}" -eq 1 ]]; then From b2bb1dcc09514617e2232939653baddb4bfbb4d1 Mon Sep 17 00:00:00 2001 From: nextdreamblue Date: Sun, 11 Dec 2022 20:08:04 +0800 Subject: [PATCH 2/6] [fix](merge-on-write) delete all rows with same key in all pre segments Signed-off-by: nextdreamblue --- be/src/olap/tablet.cpp | 52 +++++++++++++++++++++++++----------------- be/src/olap/tablet.h | 2 +- build.sh | 4 ++-- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index c670e75b4af702..7064d2f87690fc 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1999,7 +1999,7 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id, RowLocation loc; // first check if exist in pre segment if (check_pre_segments) { - auto st = _check_pk_in_pre_segments(pre_segments, *key, dummy_version, + auto st = _check_pk_in_pre_segments(rowset_id, pre_segments, *key, dummy_version, delete_bitmap, &loc); if (st.ok()) { delete_bitmap->add({loc.rowset_id, loc.segment_id, dummy_version.first}, @@ -2012,23 +2012,26 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id, continue; } } - auto st = lookup_row_key(*key, specified_rowset_ids, &loc, dummy_version.first - 1); - CHECK(st.ok() || st.is() || st.is()); - if (st.is()) { - ++row_id; - continue; - } - // sequence id smaller than the previous one, so delete current row - if (st.is()) { - loc.rowset_id = rowset_id; - loc.segment_id = seg->id(); - loc.row_id = row_id; - } + if (!specified_rowset_ids->empty()) { + auto st = lookup_row_key(*key, specified_rowset_ids, &loc, dummy_version.first - 1); + CHECK(st.ok() || st.is() || st.is()); + if (st.is()) { + ++row_id; + continue; + } + + // sequence id smaller than the previous one, so delete current row + if (st.is()) { + loc.rowset_id = rowset_id; + loc.segment_id = seg->id(); + loc.row_id = row_id; + } + delete_bitmap->add({loc.rowset_id, loc.segment_id, dummy_version.first}, + loc.row_id); + } ++row_id; - delete_bitmap->add({loc.rowset_id, loc.segment_id, dummy_version.first}, - loc.row_id); } remaining -= num_read; } @@ -2043,7 +2046,7 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id, return Status::OK(); } -Status Tablet::_check_pk_in_pre_segments( +Status Tablet::_check_pk_in_pre_segments(RowsetId rowset_id, const std::vector& pre_segments, const Slice& key, const Version& version, DeleteBitmapPtr delete_bitmap, RowLocation* loc) { for (auto it = pre_segments.rbegin(); it != pre_segments.rend(); ++it) { @@ -2052,11 +2055,19 @@ Status Tablet::_check_pk_in_pre_segments( if (st.is()) { continue; } else if (st.ok() && _schema->has_sequence_col() && - delete_bitmap->contains({loc->rowset_id, loc->segment_id, version.first}, + delete_bitmap->contains({rowset_id, loc->segment_id, version.first}, loc->row_id)) { // if has sequence col, we continue to compare the sequence_id of // all segments, util we find an existing key. continue; + } else if (st.ok() && !_schema->has_sequence_col()) { + // delete all rows with same key in all pre segments + if (!delete_bitmap->contains({rowset_id, loc->segment_id, version.first}, + loc->row_id)) { + delete_bitmap->add({rowset_id, loc->segment_id, version.first}, + loc->row_id); + } + continue; } return st; } @@ -2126,10 +2137,9 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, DeleteBitmapP for (const auto& to_del : rowset_ids_to_del) { delete_bitmap->remove({to_del, 0, 0}, {to_del, UINT32_MAX, INT64_MAX}); } - if (!rowset_ids_to_add.empty()) { - RETURN_IF_ERROR(calc_delete_bitmap(rowset->rowset_id(), segments, &rowset_ids_to_add, - delete_bitmap, cur_version - 1, true)); - } + + RETURN_IF_ERROR(calc_delete_bitmap(rowset->rowset_id(), segments, &rowset_ids_to_add, + delete_bitmap, cur_version - 1, true)); // update version without write lock, compaction and publish_txn // will update delete bitmap, handle compaction with _rowset_update_lock diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 688f98450248ff..5c61bbee532845 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -392,7 +392,7 @@ class Tablet : public BaseTablet { bool _reconstruct_version_tracker_if_necessary(); void _init_context_common_fields(RowsetWriterContext& context); - Status _check_pk_in_pre_segments(const std::vector& pre_segments, + Status _check_pk_in_pre_segments(RowsetId rowset_id, const std::vector& pre_segments, const Slice& key, const Version& version, DeleteBitmapPtr delete_bitmap, RowLocation* loc); void _rowset_ids_difference(const RowsetIdUnorderedSet& cur, const RowsetIdUnorderedSet& pre, diff --git a/build.sh b/build.sh index c977132bb39424..6ed3c5ac6f6282 100755 --- a/build.sh +++ b/build.sh @@ -83,7 +83,7 @@ clean_be() { # "build.sh --clean" just cleans and exits, however CMAKE_BUILD_DIR is set # while building be. - CMAKE_BUILD_TYPE="${BUILD_TYPE:-DEBUG}" + CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" CMAKE_BUILD_DIR="${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE}" rm -rf "${CMAKE_BUILD_DIR}" @@ -387,7 +387,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then if [[ -e "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" ]]; then rm -f "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" fi - CMAKE_BUILD_TYPE="${BUILD_TYPE:-DEBUG}" + CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" echo "Build Backend: ${CMAKE_BUILD_TYPE}" CMAKE_BUILD_DIR="${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE}" if [[ "${CLEAN}" -eq 1 ]]; then From cfda4559a74cf6692a2ce55107c7edeb51eafd85 Mon Sep 17 00:00:00 2001 From: nextdreamblue Date: Sun, 11 Dec 2022 20:16:28 +0800 Subject: [PATCH 3/6] fix code Signed-off-by: nextdreamblue --- be/src/olap/tablet.cpp | 13 ++++++------- be/src/olap/tablet.h | 3 ++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 7064d2f87690fc..455bf173555598 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1999,8 +1999,8 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id, RowLocation loc; // first check if exist in pre segment if (check_pre_segments) { - auto st = _check_pk_in_pre_segments(rowset_id, pre_segments, *key, dummy_version, - delete_bitmap, &loc); + auto st = _check_pk_in_pre_segments(rowset_id, pre_segments, *key, + dummy_version, delete_bitmap, &loc); if (st.ok()) { delete_bitmap->add({loc.rowset_id, loc.segment_id, dummy_version.first}, loc.row_id); @@ -2046,9 +2046,9 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id, return Status::OK(); } -Status Tablet::_check_pk_in_pre_segments(RowsetId rowset_id, - const std::vector& pre_segments, const Slice& key, - const Version& version, DeleteBitmapPtr delete_bitmap, RowLocation* loc) { +Status Tablet::_check_pk_in_pre_segments( + RowsetId rowset_id, const std::vector& pre_segments, + const Slice& key, const Version& version, DeleteBitmapPtr delete_bitmap, RowLocation* loc) { for (auto it = pre_segments.rbegin(); it != pre_segments.rend(); ++it) { auto st = (*it)->lookup_row_key(key, loc); CHECK(st.ok() || st.is() || st.is()); @@ -2064,8 +2064,7 @@ Status Tablet::_check_pk_in_pre_segments(RowsetId rowset_id, // delete all rows with same key in all pre segments if (!delete_bitmap->contains({rowset_id, loc->segment_id, version.first}, loc->row_id)) { - delete_bitmap->add({rowset_id, loc->segment_id, version.first}, - loc->row_id); + delete_bitmap->add({rowset_id, loc->segment_id, version.first}, loc->row_id); } continue; } diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 5c61bbee532845..38d77961bc6e4c 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -392,7 +392,8 @@ class Tablet : public BaseTablet { bool _reconstruct_version_tracker_if_necessary(); void _init_context_common_fields(RowsetWriterContext& context); - Status _check_pk_in_pre_segments(RowsetId rowset_id, const std::vector& pre_segments, + Status _check_pk_in_pre_segments(RowsetId rowset_id, + const std::vector& pre_segments, const Slice& key, const Version& version, DeleteBitmapPtr delete_bitmap, RowLocation* loc); void _rowset_ids_difference(const RowsetIdUnorderedSet& cur, const RowsetIdUnorderedSet& pre, From 8f2007e9bfa640cff7bf7c4b243718dcb4f671b9 Mon Sep 17 00:00:00 2001 From: nextdreamblue Date: Mon, 12 Dec 2022 09:24:31 +0800 Subject: [PATCH 4/6] fix code 2 Signed-off-by: nextdreamblue --- be/src/olap/tablet.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 455bf173555598..66d519d3810b64 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2014,7 +2014,8 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id, } if (!specified_rowset_ids->empty()) { - auto st = lookup_row_key(*key, specified_rowset_ids, &loc, dummy_version.first - 1); + auto st = lookup_row_key(*key, specified_rowset_ids, &loc, + dummy_version.first - 1); CHECK(st.ok() || st.is() || st.is()); if (st.is()) { ++row_id; From 175283b75b1febdef32a66dffadbe8612dcfec0b Mon Sep 17 00:00:00 2001 From: nextdreamblue Date: Mon, 12 Dec 2022 18:13:09 +0800 Subject: [PATCH 5/6] fix bug code Signed-off-by: nextdreamblue --- be/src/olap/tablet.cpp | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 66d519d3810b64..149fa5f1baa399 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2002,14 +2002,10 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id, auto st = _check_pk_in_pre_segments(rowset_id, pre_segments, *key, dummy_version, delete_bitmap, &loc); if (st.ok()) { - delete_bitmap->add({loc.rowset_id, loc.segment_id, dummy_version.first}, + delete_bitmap->add({rowset_id, loc.segment_id, dummy_version.first}, loc.row_id); - ++row_id; - continue; } else if (st.is()) { delete_bitmap->add({rowset_id, seg->id(), dummy_version.first}, row_id); - ++row_id; - continue; } } @@ -2061,13 +2057,6 @@ Status Tablet::_check_pk_in_pre_segments( // if has sequence col, we continue to compare the sequence_id of // all segments, util we find an existing key. continue; - } else if (st.ok() && !_schema->has_sequence_col()) { - // delete all rows with same key in all pre segments - if (!delete_bitmap->contains({rowset_id, loc->segment_id, version.first}, - loc->row_id)) { - delete_bitmap->add({rowset_id, loc->segment_id, version.first}, loc->row_id); - } - continue; } return st; } From 5edb6292a92c1ac28f1bc091888caa3a3f7b0455 Mon Sep 17 00:00:00 2001 From: nextdreamblue Date: Mon, 12 Dec 2022 19:58:32 +0800 Subject: [PATCH 6/6] fix code 2 Signed-off-by: nextdreamblue --- be/src/olap/tablet.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 149fa5f1baa399..4fed91ed156572 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2004,8 +2004,12 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id, if (st.ok()) { delete_bitmap->add({rowset_id, loc.segment_id, dummy_version.first}, loc.row_id); + ++row_id; + continue; } else if (st.is()) { delete_bitmap->add({rowset_id, seg->id(), dummy_version.first}, row_id); + ++row_id; + continue; } }