From 803e2a97cf2e8d37e023a4be1829906f04dbddbe Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Thu, 11 Apr 2024 16:17:11 +0800 Subject: [PATCH 1/3] [fix](merge-on-write) schema change may cause mow duplicate key --- be/src/olap/schema_change.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 53c23344282576..05467642f2957f 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -935,6 +935,7 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& } int64_t real_alter_version = 0; res = _convert_historical_rowsets(sc_params, &real_alter_version); + DCHECK_GE(real_alter_version, request.alter_version); { std::lock_guard wrlock(_mutex); _tablet_ids_in_converting.erase(new_tablet->tablet_id()); @@ -1061,6 +1062,7 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams auto sc_procedure = get_sc_procedure(changer, sc_sorting, sc_directly); // c.Convert historical data + bool have_failure_rowset = false; for (auto& rs_reader : sc_params.ref_rowset_readers) { VLOG_TRACE << "begin to convert a history rowset. version=" << rs_reader->version().first << "-" << rs_reader->version().second; @@ -1113,6 +1115,8 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams << "tablet=" << sc_params.new_tablet->full_name() << ", version='" << rs_reader->version().first << "-" << rs_reader->version().second; StorageEngine::instance()->add_unused_rowset(new_rowset); + have_failure_rowset = true; + _local_storage_engine.add_unused_rowset(new_rowset); res = Status::OK(); } else if (!res) { LOG(WARNING) << "failed to register new version. " @@ -1126,7 +1130,9 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams << ", version=" << rs_reader->version().first << "-" << rs_reader->version().second; } - *real_alter_version = rs_reader->version().second; + if (!have_failure_rowset) { + *real_alter_version = rs_reader->version().second; + } VLOG_TRACE << "succeed to convert a history version." << " version=" << rs_reader->version().first << "-" From 5e5efd6ef4a2c7c37454d1b50d1e76b64a90d63b Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Thu, 11 Apr 2024 16:53:43 +0800 Subject: [PATCH 2/3] fix cp error --- be/src/olap/schema_change.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 05467642f2957f..6c253614922c28 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1116,7 +1116,6 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams << rs_reader->version().first << "-" << rs_reader->version().second; StorageEngine::instance()->add_unused_rowset(new_rowset); have_failure_rowset = true; - _local_storage_engine.add_unused_rowset(new_rowset); res = Status::OK(); } else if (!res) { LOG(WARNING) << "failed to register new version. " From 2d9d89de21840ea82ebb64764f964b1245e4ef2d Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Fri, 12 Apr 2024 20:52:27 +0800 Subject: [PATCH 3/3] update DCHECK --- be/src/olap/schema_change.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 6c253614922c28..84ecbb10c60879 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -935,7 +935,6 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& } int64_t real_alter_version = 0; res = _convert_historical_rowsets(sc_params, &real_alter_version); - DCHECK_GE(real_alter_version, request.alter_version); { std::lock_guard wrlock(_mutex); _tablet_ids_in_converting.erase(new_tablet->tablet_id()); @@ -944,6 +943,8 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& break; } + DCHECK_GE(real_alter_version, request.alter_version); + if (new_tablet->keys_type() == UNIQUE_KEYS && new_tablet->enable_unique_key_merge_on_write()) { res = _calc_delete_bitmap_for_mow_table(new_tablet, real_alter_version);