From 3e3009c29be936859cd42fa338672d526587f24f Mon Sep 17 00:00:00 2001 From: deardeng <565620795@qq.com> Date: Tue, 19 Dec 2023 11:49:01 +0800 Subject: [PATCH 1/2] [fix](partition) Comment out some exceptions with partition equal to 0 and return In some scenarios, the partition id in the table meta is 0, but the rowset meta partition is not 0, (#28055) PR check partition id will caused be failed to start when upgrade meta like this ``` { "table_id": 1266302, "partition_id": 0, "tablet_id": 1289222, "schema_hash": 816078096, "shard_id": 949, "creation_time": 1684805433, "cumulative_layer_point": -1, "tablet_state": "PB_RUNNING", "schema": { "keys_type": "DUP_KEYS", "column": [ ... ], "num_short_key_columns": 1, "num_rows_per_row_block": 1024, ... "skip_write_index_on_load": false }, "rs_metas": [ { "rowset_id": 0, "partition_id": 1289193, "tablet_id": 1289222, "tablet_schema_hash": 816078096, "newest_write_timestamp": 1684805433, ... "segments_overlap_pb": "NONOVERLAPPING" } ], "in_restore_mode": false, "tablet_uid": { "hi": -4301918259617109775, "lo": 2820124814688031668 }, "preferred_rowset_type": "BETA_ROWSET", "tablet_type": "TABLET_TYPE_DISK", "replica_id": 1812028, "enable_unique_key_merge_on_write": false, "binlog_config": { "enable": false, "ttl_seconds": 9223372036854775807, "max_bytes": 9223372036854775807, "max_history_nums": 9223372036854775807 }, "compaction_policy": "size_based", "time_series_compaction_goal_size_mbytes": 1024, "time_series_compaction_file_count_threshold": 2000, "time_series_compaction_time_threshold_seconds": 3600 } ``` --- be/src/olap/rowset/rowset_meta_manager.cpp | 5 +++-- be/src/olap/tablet_manager.cpp | 12 ++++++++++++ be/src/olap/tablet_meta_manager.cpp | 5 +++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp b/be/src/olap/rowset/rowset_meta_manager.cpp index 01384a560575d5..be49e66ee1114d 100644 --- a/be/src/olap/rowset/rowset_meta_manager.cpp +++ b/be/src/olap/rowset/rowset_meta_manager.cpp @@ -91,8 +91,9 @@ Status RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, const Rowse if (rowset_meta_pb.partition_id() <= 0) { LOG(WARNING) << "invalid partition id " << rowset_meta_pb.partition_id() << " tablet " << rowset_meta_pb.tablet_id(); - return Status::InternalError("invalid partition id {}, tablet {}", - rowset_meta_pb.partition_id(), rowset_meta_pb.tablet_id()); + // TODO(dx): after fix partition id eq 0 bug, fix it + // return Status::InternalError("invaid partition id {} tablet {}", + // rowset_meta_pb.partition_id(), rowset_meta_pb.tablet_id()); } if (enable_binlog) { return _save_with_binlog(meta, tablet_uid, rowset_id, rowset_meta_pb); diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index ee060d0fed76ea..500273baa09202 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -501,6 +501,12 @@ TabletSharedPtr TabletManager::_create_tablet_meta_and_dir_unlocked( } } + if (tablet_meta->partition_id() <= 0) { + LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << ", tablet " + << tablet_meta->tablet_id(); + // TODO(dx): after fix partition id eq 0 bug, fix it + // return nullptr; + } TabletSharedPtr new_tablet = std::make_shared(std::move(tablet_meta), data_dir); COUNTER_UPDATE(ADD_CHILD_TIMER(profile, "CreateTabletFromMeta", parent_timer_name), static_cast(watch.reset())); @@ -826,6 +832,12 @@ Status TabletManager::load_tablet_from_meta(DataDir* data_dir, TTabletId tablet_ tablet_meta->set_tablet_state(TABLET_RUNNING); } + if (tablet_meta->partition_id() <= 0) { + LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << ", tablet " + << tablet_meta->tablet_id(); + // TODO(dx): after fix partition id eq 0 bug, fix it + // return nullptr; + } TabletSharedPtr tablet = std::make_shared(std::move(tablet_meta), data_dir); // NOTE: method load_tablet_from_meta could be called by two cases as below diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp index 17d5d2e9f74b09..288da455e6a888 100644 --- a/be/src/olap/tablet_meta_manager.cpp +++ b/be/src/olap/tablet_meta_manager.cpp @@ -96,8 +96,9 @@ Status TabletMetaManager::save(DataDir* store, TTabletId tablet_id, TSchemaHash if (tablet_meta->partition_id() <= 0) { LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << " tablet " << tablet_meta->tablet_id(); - return Status::InternalError("invaid partition id {} tablet {}", - tablet_meta->partition_id(), tablet_meta->tablet_id()); + // TODO(dx): after fix partition id eq 0 bug, fix it + // return Status::InternalError("invaid partition id {} tablet {}", + // tablet_meta->partition_id(), tablet_meta->tablet_id()); } OlapMeta* meta = store->get_meta(); VLOG_NOTICE << "save tablet meta" From 3520263c93f610c26068a9c31fd8f05983c729c1 Mon Sep 17 00:00:00 2001 From: deardeng <565620795@qq.com> Date: Wed, 20 Dec 2023 16:08:47 +0800 Subject: [PATCH 2/2] fix --- be/src/olap/tablet_manager.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 500273baa09202..f1961d8ee9d13b 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -503,9 +503,7 @@ TabletSharedPtr TabletManager::_create_tablet_meta_and_dir_unlocked( if (tablet_meta->partition_id() <= 0) { LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << ", tablet " - << tablet_meta->tablet_id(); - // TODO(dx): after fix partition id eq 0 bug, fix it - // return nullptr; + << tablet_meta->tablet_id(); } TabletSharedPtr new_tablet = std::make_shared(std::move(tablet_meta), data_dir); COUNTER_UPDATE(ADD_CHILD_TIMER(profile, "CreateTabletFromMeta", parent_timer_name), @@ -834,9 +832,7 @@ Status TabletManager::load_tablet_from_meta(DataDir* data_dir, TTabletId tablet_ if (tablet_meta->partition_id() <= 0) { LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << ", tablet " - << tablet_meta->tablet_id(); - // TODO(dx): after fix partition id eq 0 bug, fix it - // return nullptr; + << tablet_meta->tablet_id(); } TabletSharedPtr tablet = std::make_shared(std::move(tablet_meta), data_dir);