From f9175486b183c0cc8a10c64ebc64d1b244efefc4 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 6 Mar 2025 15:53:55 +0800 Subject: [PATCH 1/4] add reproduce case --- ...loud_multi_segments_re_calc_in_publish.out | 9 ++ ...d_multi_segments_re_calc_in_publish.groovy | 144 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out create mode 100644 regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy diff --git a/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out b/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out new file mode 100644 index 00000000000000..ccd9979c8557e1 --- /dev/null +++ b/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +77777 77777 77777 +88888 88888 88888 +99999 99999 99999 + +-- !dup_key_count -- +4096 + diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy new file mode 100644 index 00000000000000..946d3a1e6bb54c --- /dev/null +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_cloud_multi_segments_re_calc_in_publish", "nonConcurrent") { + if (!isCloudMode()) { + return + } + + def table1 = "test_cloud_multi_segments_re_calc_in_publish" + sql "DROP TABLE IF EXISTS ${table1} FORCE;" + sql """ CREATE TABLE IF NOT EXISTS ${table1} ( + `k1` int NOT NULL, + `c1` int, + `c2` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1"); """ + + sql "insert into ${table1} values(99999,99999,99999);" + sql "insert into ${table1} values(88888,88888,88888);" + sql "insert into ${table1} values(77777,77777,77777);" + sql "sync;" + qt_sql "select * from ${table1} order by k1;" + + def block_publish = { + if (isCloudMode()) { + GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait") + GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block") + } else { + GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait") + GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.block") + + } + } + + def unblock_publish = { + if (isCloudMode()) { + GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait") + GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block") + } else { + GetDebugPoint().disableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait") + GetDebugPoint().disableDebugPointForAllBEs("EnginePublishVersionTask::execute.block") + + } + } + + def checkSegmentNum = { rowsetNum, lastRowsetSegmentNum -> + def tablets = sql_return_maparray """ show tablets from ${table1}; """ + logger.info("tablets: ${tablets}") + String compactionUrl = tablets[0]["CompactionStatus"] + def (code, out, err) = curl("GET", compactionUrl) + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + assert tabletJson.rowsets.size() == rowsetNum + 1 + def rowset = tabletJson.rowsets.get(tabletJson.rowsets.size() - 1) + logger.info("rowset: ${rowset}") + int start_index = rowset.indexOf("]") + int end_index = rowset.indexOf("DATA") + def segmentNumStr = rowset.substring(start_index + 1, end_index).trim() + logger.info("segmentNumStr: ${segmentNumStr}") + assert lastRowsetSegmentNum == Integer.parseInt(segmentNumStr) + } + + // to cause multi segments + def customBeConfig = [ + doris_scanner_row_bytes : 1 + ] + + setBeConfigTemporary(customBeConfig) { + try { + // batch_size is 4164 in csv_reader.cpp + // _batch_size is 8192 in vtablet_writer.cpp + // to cause multi segments + GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush") + + // block_publish() + + // inject cache miss so that it will re-calculate delete bitmaps for all historical data in publish + GetDebugPoint().enableDebugPointForAllBEs("CloudTxnDeleteBitmapCache::get_delete_bitmap.cache_miss") + + Thread.sleep(1000) + + def t1 = Thread.start { + // load data that will have multi segments and there are duplicate keys between segments + String content = "" + (1..4096).each { + content += "${it},${it},${it}\n" + } + content += content + streamLoad { + table "${table1}" + set 'column_separator', ',' + inputStream new ByteArrayInputStream(content.getBytes()) + time 30000 // limit inflight 10s + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(result) + assert "success" == json.Status.toLowerCase() + assert 8192 == json.NumberTotalRows + assert 0 == json.NumberFilteredRows + } + } + } + + + t1.join() + Thread.sleep(1000) + + // ensure that we really write multi segments + checkSegmentNum(4, 3) + + qt_dup_key_count "select count() from (select k1,count() as cnt from ${table1} group by k1 having cnt > 1) A;" + + } catch(Exception e) { + logger.info(e.getMessage()) + throw e + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + } + } +} From 4c1d42e447be87136eb853c48d3db9efb8076b92 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 6 Mar 2025 16:53:16 +0800 Subject: [PATCH 2/4] fix and correct case --- .../cloud_engine_calc_delete_bitmap_task.cpp | 9 ++++++ ...loud_multi_segments_re_calc_in_publish.out | 5 ++- ...d_multi_segments_re_calc_in_publish.groovy | 32 ++++--------------- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp index f4e25302374326..b5cb220a087333 100644 --- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp +++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp @@ -29,6 +29,7 @@ #include "common/status.h" #include "olap/base_tablet.h" #include "olap/olap_common.h" +#include "olap/rowset/beta_rowset.h" #include "olap/rowset/rowset.h" #include "olap/tablet_fwd.h" #include "olap/tablet_meta.h" @@ -289,6 +290,14 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset( return status; } + if (rowset_ids.empty()) { + // delete bitmap cache missed, should re-calculate delete bitmaps between segments + std::vector segments; + RETURN_IF_ERROR(std::static_pointer_cast(rowset)->load_segments(&segments)); + RETURN_IF_ERROR(tablet->calc_delete_bitmap_between_segments(rowset->rowset_id(), segments, + delete_bitmap)); + } + rowset->set_version(Version(version, version)); TabletTxnInfo txn_info; txn_info.rowset = rowset; diff --git a/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out b/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out index ccd9979c8557e1..f9e767c3d20f3e 100644 --- a/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out +++ b/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out @@ -4,6 +4,9 @@ 88888 88888 88888 99999 99999 99999 +-- !sql -- +4099 + -- !dup_key_count -- -4096 +0 diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy index 946d3a1e6bb54c..bb43c08abff0ef 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy @@ -39,28 +39,6 @@ suite("test_cloud_multi_segments_re_calc_in_publish", "nonConcurrent") { sql "sync;" qt_sql "select * from ${table1} order by k1;" - def block_publish = { - if (isCloudMode()) { - GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait") - GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block") - } else { - GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait") - GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.block") - - } - } - - def unblock_publish = { - if (isCloudMode()) { - GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait") - GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block") - } else { - GetDebugPoint().disableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait") - GetDebugPoint().disableDebugPointForAllBEs("EnginePublishVersionTask::execute.block") - - } - } - def checkSegmentNum = { rowsetNum, lastRowsetSegmentNum -> def tablets = sql_return_maparray """ show tablets from ${table1}; """ logger.info("tablets: ${tablets}") @@ -92,8 +70,6 @@ suite("test_cloud_multi_segments_re_calc_in_publish", "nonConcurrent") { // to cause multi segments GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush") - // block_publish() - // inject cache miss so that it will re-calculate delete bitmaps for all historical data in publish GetDebugPoint().enableDebugPointForAllBEs("CloudTxnDeleteBitmapCache::get_delete_bitmap.cache_miss") @@ -126,10 +102,14 @@ suite("test_cloud_multi_segments_re_calc_in_publish", "nonConcurrent") { t1.join() - Thread.sleep(1000) + Thread.sleep(2000) + + GetDebugPoint().clearDebugPointsForAllBEs() // ensure that we really write multi segments - checkSegmentNum(4, 3) + // checkSegmentNum(4, 3) + + qt_sql "select count() from ${table1};" qt_dup_key_count "select count() from (select k1,count() as cnt from ${table1} group by k1 having cnt > 1) A;" From 0066eba0de2e97fe7eddeb8d1e4657a3d99c4d4f Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 7 Mar 2025 10:20:58 +0800 Subject: [PATCH 3/4] fix --- .../cloud_engine_calc_delete_bitmap_task.cpp | 16 ++++++++-------- ...loud_multi_segments_re_calc_in_publish.groovy | 7 +++---- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp index b5cb220a087333..1358169f9a4afb 100644 --- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp +++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp @@ -290,14 +290,6 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset( return status; } - if (rowset_ids.empty()) { - // delete bitmap cache missed, should re-calculate delete bitmaps between segments - std::vector segments; - RETURN_IF_ERROR(std::static_pointer_cast(rowset)->load_segments(&segments)); - RETURN_IF_ERROR(tablet->calc_delete_bitmap_between_segments(rowset->rowset_id(), segments, - delete_bitmap)); - } - rowset->set_version(Version(version, version)); TabletTxnInfo txn_info; txn_info.rowset = rowset; @@ -327,6 +319,14 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset( LOG(INFO) << "tablet=" << _tablet_id << ", " << txn_str << ", publish_status=SUCCEED, not need to re-calculate delete_bitmaps."; } else { + if (rowset_ids.empty()) { + // delete bitmap cache missed, should re-calculate delete bitmaps between segments + std::vector segments; + RETURN_IF_ERROR(std::static_pointer_cast(rowset)->load_segments(&segments)); + RETURN_IF_ERROR(tablet->calc_delete_bitmap_between_segments(rowset->rowset_id(), + segments, delete_bitmap)); + } + if (invisible_rowsets == nullptr) { status = CloudTablet::update_delete_bitmap(tablet, &txn_info, transaction_id, txn_expiration); diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy index bb43c08abff0ef..b741a6e998632b 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy @@ -102,17 +102,16 @@ suite("test_cloud_multi_segments_re_calc_in_publish", "nonConcurrent") { t1.join() - Thread.sleep(2000) GetDebugPoint().clearDebugPointsForAllBEs() - - // ensure that we really write multi segments - // checkSegmentNum(4, 3) + Thread.sleep(2000) qt_sql "select count() from ${table1};" qt_dup_key_count "select count() from (select k1,count() as cnt from ${table1} group by k1 having cnt > 1) A;" + // ensure that we really write multi segments + checkSegmentNum(4, 3) } catch(Exception e) { logger.info(e.getMessage()) throw e From 00d6e46142577e147b69a37f6f991ea6530b1679 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 7 Mar 2025 11:49:12 +0800 Subject: [PATCH 4/4] use marks to determine --- be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp index 1358169f9a4afb..06ebf249edb0ef 100644 --- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp +++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp @@ -319,7 +319,8 @@ Status CloudTabletCalcDeleteBitmapTask::_handle_rowset( LOG(INFO) << "tablet=" << _tablet_id << ", " << txn_str << ", publish_status=SUCCEED, not need to re-calculate delete_bitmaps."; } else { - if (rowset_ids.empty()) { + if (rowset->num_segments() > 1 && + !delete_bitmap->has_calculated_for_multi_segments(rowset->rowset_id())) { // delete bitmap cache missed, should re-calculate delete bitmaps between segments std::vector segments; RETURN_IF_ERROR(std::static_pointer_cast(rowset)->load_segments(&segments));