From 0fc2f8a83e80a99798a53c1ce61b246cb997dc1a Mon Sep 17 00:00:00 2001 From: lambxu Date: Mon, 1 Jul 2024 18:28:08 +0800 Subject: [PATCH 1/4] save --- be/src/olap/single_replica_compaction.cpp | 4 ++-- be/src/olap/snapshot_manager.cpp | 10 ++++++++-- be/src/olap/snapshot_manager.h | 1 + be/src/olap/task/engine_clone_task.cpp | 2 +- be/src/olap/task/engine_storage_migration_task.cpp | 2 +- be/src/runtime/snapshot_loader.cpp | 2 +- .../src/main/java/org/apache/doris/task/CloneTask.java | 2 +- gensrc/thrift/AgentService.thrift | 1 + 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/be/src/olap/single_replica_compaction.cpp b/be/src/olap/single_replica_compaction.cpp index 571156e721c749..dfdc0132eeccfe 100644 --- a/be/src/olap/single_replica_compaction.cpp +++ b/be/src/olap/single_replica_compaction.cpp @@ -314,8 +314,8 @@ Status SingleReplicaCompaction::_fetch_rowset(const TReplicaInfo& addr, const st } RETURN_IF_ERROR(_download_files(tablet()->data_dir(), remote_url_prefix, local_path)); _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( - local_path, _tablet->tablet_id(), tablet()->replica_id(), _tablet->partition_id(), - _tablet->schema_hash())); + local_path, _tablet->tablet_id(), tablet()->replica_id(), _tablet->table_id(), + _tablet->partition_id(), _tablet->schema_hash())); // 4: finish_clone: create output_rowset and link file return _finish_clone(local_data_path, rowset_version); } diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index eab49997599ce4..54ca71016ea443 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -120,8 +120,8 @@ Status SnapshotManager::release_snapshot(const string& snapshot_path) { } Result> SnapshotManager::convert_rowset_ids( - const std::string& clone_dir, int64_t tablet_id, int64_t replica_id, int64_t partition_id, - int32_t schema_hash) { + const std::string& clone_dir, int64_t tablet_id, int64_t replica_id, int64_t table_id, + int64_t partition_id, int32_t schema_hash) { SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker); std::vector guards; // check clone dir existed @@ -151,6 +151,9 @@ Result> SnapshotManager::convert_rowset_ids( new_tablet_meta_pb.set_tablet_id(tablet_id); *new_tablet_meta_pb.mutable_tablet_uid() = TabletUid::gen_uid().to_proto(); new_tablet_meta_pb.set_replica_id(replica_id); + if (table_id > 0) { + new_tablet_meta_pb.set_table_id(table_id); + } if (partition_id != -1) { new_tablet_meta_pb.set_partition_id(partition_id); } @@ -183,6 +186,9 @@ Result> SnapshotManager::convert_rowset_ids( } rowset_meta->set_tablet_id(tablet_id); + if (table_id > 0) { + rowset_meta.set_table_id(table_id); + } if (partition_id != -1) { rowset_meta->set_partition_id(partition_id); } diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h index 74225c7976d4e7..dd8b9e168c5eda 100644 --- a/be/src/olap/snapshot_manager.h +++ b/be/src/olap/snapshot_manager.h @@ -54,6 +54,7 @@ class SnapshotManager { Result> convert_rowset_ids(const std::string& clone_dir, int64_t tablet_id, int64_t replica_id, + int64_t table_id, int64_t partition_id, int32_t schema_hash); diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index 590fdd1b4a7449..2d279ad0a69b10 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -421,7 +421,7 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir, // No need to try again with another BE _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( local_data_path, _clone_req.tablet_id, _clone_req.replica_id, - _clone_req.partition_id, _clone_req.schema_hash)); + _clone_req.table_id, _clone_req.partition_id, _clone_req.schema_hash)); break; } // clone copy from one backend return status; diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index 39926d334600e5..65213f1996011f 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -159,7 +159,7 @@ Status EngineStorageMigrationTask::_gen_and_write_header_to_hdr_file( // it will change rowset id and its create time // rowset create time is useful when load tablet from meta to check which tablet is the tablet to load _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( - full_path, tablet_id, _tablet->replica_id(), _tablet->partition_id(), schema_hash)); + full_path, tablet_id, _tablet->replica_id(), _tablet->table_id(), _tablet->partition_id(), schema_hash)); return Status::OK(); } diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index a742e934ec31f8..504d540562a4e4 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -754,7 +754,7 @@ Status SnapshotLoader::move(const std::string& snapshot_path, TabletSharedPtr ta // rename the rowset ids and tabletid info in rowset meta auto res = _engine.snapshot_mgr()->convert_rowset_ids( - snapshot_path, tablet_id, tablet->replica_id(), tablet->partition_id(), schema_hash); + snapshot_path, tablet_id, tablet->replica_id(), tablet->table_id(), tablet->partition_id(), schema_hash); if (!res.has_value()) [[unlikely]] { auto err_msg = fmt::format("failed to convert rowsetids in snapshot: {}, tablet path: {}, err: {}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java index 891ef99c6ed948..d1c1ed177345fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CloneTask.java @@ -91,7 +91,7 @@ public TCloneReq toThrift() { request.setDestPathHash(destPathHash); } request.setTimeoutS(timeoutS); - + request.setTableId(tableId); return request; } diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index a03cb9df99b69f..87525fcca0c11b 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -309,6 +309,7 @@ struct TCloneReq { 10: optional i32 timeout_s; 11: optional Types.TReplicaId replica_id = 0 12: optional i64 partition_id + 13: optional i64 table_id } struct TCompactionReq { From bc62ce087c5e2cc3b8d6dd38614f1e617e9e91e9 Mon Sep 17 00:00:00 2001 From: lambxu Date: Mon, 1 Jul 2024 18:35:48 +0800 Subject: [PATCH 2/4] formta --- be/src/olap/snapshot_manager.h | 3 +-- be/src/olap/task/engine_clone_task.cpp | 4 ++-- be/src/olap/task/engine_storage_migration_task.cpp | 3 ++- be/src/runtime/snapshot_loader.cpp | 5 +++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h index dd8b9e168c5eda..df2b1b33b271eb 100644 --- a/be/src/olap/snapshot_manager.h +++ b/be/src/olap/snapshot_manager.h @@ -53,8 +53,7 @@ class SnapshotManager { Result> convert_rowset_ids(const std::string& clone_dir, int64_t tablet_id, - int64_t replica_id, - int64_t table_id, + int64_t replica_id, int64_t table_id, int64_t partition_id, int32_t schema_hash); diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index 2d279ad0a69b10..2fc50635d723ff 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -420,8 +420,8 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir, } // No need to try again with another BE _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( - local_data_path, _clone_req.tablet_id, _clone_req.replica_id, - _clone_req.table_id, _clone_req.partition_id, _clone_req.schema_hash)); + local_data_path, _clone_req.tablet_id, _clone_req.replica_id, _clone_req.table_id, + _clone_req.partition_id, _clone_req.schema_hash)); break; } // clone copy from one backend return status; diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index 65213f1996011f..7c870a5e8ea5fc 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -159,7 +159,8 @@ Status EngineStorageMigrationTask::_gen_and_write_header_to_hdr_file( // it will change rowset id and its create time // rowset create time is useful when load tablet from meta to check which tablet is the tablet to load _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( - full_path, tablet_id, _tablet->replica_id(), _tablet->table_id(), _tablet->partition_id(), schema_hash)); + full_path, tablet_id, _tablet->replica_id(), _tablet->table_id(), + _tablet->partition_id(), schema_hash)); return Status::OK(); } diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index 504d540562a4e4..b840636a46f0fb 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -753,8 +753,9 @@ Status SnapshotLoader::move(const std::string& snapshot_path, TabletSharedPtr ta } // rename the rowset ids and tabletid info in rowset meta - auto res = _engine.snapshot_mgr()->convert_rowset_ids( - snapshot_path, tablet_id, tablet->replica_id(), tablet->table_id(), tablet->partition_id(), schema_hash); + auto res = _engine.snapshot_mgr()->convert_rowset_ids(snapshot_path, tablet_id, + tablet->replica_id(), tablet->table_id(), + tablet->partition_id(), schema_hash); if (!res.has_value()) [[unlikely]] { auto err_msg = fmt::format("failed to convert rowsetids in snapshot: {}, tablet path: {}, err: {}", From 2e47b9608571f8a73332fe55ae1998278159d619 Mon Sep 17 00:00:00 2001 From: lambxu Date: Mon, 1 Jul 2024 18:45:12 +0800 Subject: [PATCH 3/4] save --- be/src/olap/snapshot_manager.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index 54ca71016ea443..31dc0d4d2e8456 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -186,9 +186,6 @@ Result> SnapshotManager::convert_rowset_ids( } rowset_meta->set_tablet_id(tablet_id); - if (table_id > 0) { - rowset_meta.set_table_id(table_id); - } if (partition_id != -1) { rowset_meta->set_partition_id(partition_id); } From 4b6ce76e8a8fb28ad2b9e90a2c42c9f2035ddf07 Mon Sep 17 00:00:00 2001 From: lambxu Date: Tue, 2 Jul 2024 13:00:52 +0800 Subject: [PATCH 4/4] save --- gensrc/thrift/AgentService.thrift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 87525fcca0c11b..289aabb971f036 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -309,7 +309,7 @@ struct TCloneReq { 10: optional i32 timeout_s; 11: optional Types.TReplicaId replica_id = 0 12: optional i64 partition_id - 13: optional i64 table_id + 13: optional i64 table_id = -1 } struct TCompactionReq {