From 921c9e5d78a8f2c14e1b72a81d8e0613b718c41e Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Thu, 11 Apr 2024 16:17:11 +0800 Subject: [PATCH 1/2] [fix](merge-on-write) schema change may cause mow duplicate key --- be/src/olap/schema_change.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index fb9137d4c84eac..3f222aa2abc06f 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -961,6 +961,7 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques sc_params.enable_unique_key_merge_on_write = _new_tablet->enable_unique_key_merge_on_write(); res = _convert_historical_rowsets(sc_params, &real_alter_version); + DCHECK_GE(real_alter_version, request.alter_version); { std::lock_guard wrlock(_mutex); _tablet_ids_in_converting.erase(_new_tablet->tablet_id()); @@ -1086,6 +1087,7 @@ Status SchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParams& sc auto sc_procedure = _get_sc_procedure(changer, sc_sorting, sc_directly); // c.Convert historical data + bool have_failure_rowset = false; for (const auto& rs_reader : sc_params.ref_rowset_readers) { // set status for monitor // As long as there is a new_table as running, ref table is set as running @@ -1131,6 +1133,7 @@ Status SchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParams& sc << "tablet=" << _new_tablet->tablet_id() << ", version='" << rs_reader->version().first << "-" << rs_reader->version().second; _local_storage_engine.add_unused_rowset(new_rowset); + have_failure_rowset = true; res = Status::OK(); } else if (!res) { LOG(WARNING) << "failed to register new version. " @@ -1144,7 +1147,9 @@ Status SchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParams& sc << ", version=" << rs_reader->version().first << "-" << rs_reader->version().second; } - *real_alter_version = rs_reader->version().second; + if (!have_failure_rowset) { + *real_alter_version = rs_reader->version().second; + } VLOG_TRACE << "succeed to convert a history version." << " version=" << rs_reader->version().first << "-" From 9f8b878541a5642d97d5d40609545bfc738306e3 Mon Sep 17 00:00:00 2001 From: Chen Zhang Date: Fri, 12 Apr 2024 20:49:28 +0800 Subject: [PATCH 2/2] fix case fail --- be/src/olap/schema_change.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 3f222aa2abc06f..13182467591a5e 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -961,7 +961,6 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques sc_params.enable_unique_key_merge_on_write = _new_tablet->enable_unique_key_merge_on_write(); res = _convert_historical_rowsets(sc_params, &real_alter_version); - DCHECK_GE(real_alter_version, request.alter_version); { std::lock_guard wrlock(_mutex); _tablet_ids_in_converting.erase(_new_tablet->tablet_id()); @@ -970,6 +969,8 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques break; } + DCHECK_GE(real_alter_version, request.alter_version); + if (_new_tablet->keys_type() == UNIQUE_KEYS && _new_tablet->enable_unique_key_merge_on_write()) { res = _calc_delete_bitmap_for_mow_table(real_alter_version);