From 63bb159d1eb9e49f16775fa6d71d252eb60795a4 Mon Sep 17 00:00:00 2001 From: chaoyli Date: Thu, 13 Jun 2019 15:27:47 +0800 Subject: [PATCH 1/5] Add copy function to support storage migration task --- be/src/olap/rowset/alpha_rowset.cpp | 18 ++++++++-- be/src/olap/rowset/alpha_rowset.h | 4 ++- be/src/olap/rowset/alpha_rowset_writer.cpp | 2 +- be/src/olap/rowset/rowset.h | 4 ++- be/src/olap/rowset/segment_group.cpp | 35 ++++++++++++++++++- be/src/olap/rowset/segment_group.h | 4 ++- .../task/engine_storage_migration_task.cpp | 12 ++++--- 7 files changed, 68 insertions(+), 11 deletions(-) diff --git a/be/src/olap/rowset/alpha_rowset.cpp b/be/src/olap/rowset/alpha_rowset.cpp index be6c31d300e265..97642e0a489c8d 100644 --- a/be/src/olap/rowset/alpha_rowset.cpp +++ b/be/src/olap/rowset/alpha_rowset.cpp @@ -207,9 +207,9 @@ int64_t AlphaRowset::ref_count() const { } OLAPStatus AlphaRowset::make_snapshot(const std::string& snapshot_path, - std::vector* success_files) { + std::vector* success_links) { for (auto& segment_group : _segment_groups) { - OLAPStatus status = segment_group->make_snapshot(snapshot_path, success_files); + OLAPStatus status = segment_group->make_snapshot(snapshot_path, success_links); if (status != OLAP_SUCCESS) { LOG(WARNING) << "create hard links failed for segment group:" << segment_group->segment_group_id(); @@ -219,6 +219,20 @@ OLAPStatus AlphaRowset::make_snapshot(const std::string& snapshot_path, return OLAP_SUCCESS; } +OLAPStatus AlphaRowset::copy_files_to_path(const std::string& dest_path, + std::vector* success_files) { + for (auto& segment_group : _segment_groups) { + OLAPStatus status = segment_group->copy_files_to_path(dest_path, success_files); + if (status != OLAP_SUCCESS) { + LOG(WARNING) << "copy files failed for segment group." + << " segment_group_id:" << segment_group->segment_group_id() + << ", dest_path:" << dest_path; + return status; + } + } + return OLAP_SUCCESS; +} + OLAPStatus AlphaRowset::convert_from_old_files(const std::string& snapshot_path, std::vector* success_files) { for (auto& segment_group : _segment_groups) { diff --git a/be/src/olap/rowset/alpha_rowset.h b/be/src/olap/rowset/alpha_rowset.h index fb73b82adf7701..b50acd233b3bb4 100644 --- a/be/src/olap/rowset/alpha_rowset.h +++ b/be/src/olap/rowset/alpha_rowset.h @@ -85,7 +85,9 @@ class AlphaRowset : public Rowset { int64_t ref_count() const override; OLAPStatus make_snapshot(const std::string& snapshot_path, - std::vector* success_files) override; + std::vector* success_links) override; + OLAPStatus copy_files_to_path(const std::string& dest_path, + std::vector* success_files); OLAPStatus convert_from_old_files(const std::string& snapshot_path, std::vector* success_files); diff --git a/be/src/olap/rowset/alpha_rowset_writer.cpp b/be/src/olap/rowset/alpha_rowset_writer.cpp index 22040c06ed8c45..3e607e46243bf2 100644 --- a/be/src/olap/rowset/alpha_rowset_writer.cpp +++ b/be/src/olap/rowset/alpha_rowset_writer.cpp @@ -125,7 +125,7 @@ OLAPStatus AlphaRowsetWriter::add_rowset(RowsetSharedPtr rowset) { AlphaRowsetSharedPtr alpha_rowset = std::dynamic_pointer_cast(rowset); for (auto& segment_group : alpha_rowset->_segment_groups) { RETURN_NOT_OK(_init()); - RETURN_NOT_OK(segment_group->copy_segments_to_path(_rowset_writer_context.rowset_path_prefix, + RETURN_NOT_OK(segment_group->link_segments_to_path(_rowset_writer_context.rowset_path_prefix, _rowset_writer_context.rowset_id)); _cur_segment_group->set_empty(segment_group->empty()); _cur_segment_group->set_num_segments(segment_group->num_segments()); diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index 065d58b085a131..16513ad30aaeba 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -99,7 +99,9 @@ class Rowset : public std::enable_shared_from_this { virtual int64_t ref_count() const = 0; virtual OLAPStatus make_snapshot(const std::string& snapshot_path, - std::vector* success_files) = 0; + std::vector* success_links) = 0; + virtual OLAPStatus copy_files_to_path(const std::string& dest_path, + std::vector* success_files) = 0; virtual OLAPStatus remove_old_files(std::vector* files_to_remove) = 0; diff --git a/be/src/olap/rowset/segment_group.cpp b/be/src/olap/rowset/segment_group.cpp index ad37f14c3047b8..8b3e3811bfa6e2 100644 --- a/be/src/olap/rowset/segment_group.cpp +++ b/be/src/olap/rowset/segment_group.cpp @@ -723,6 +723,39 @@ OLAPStatus SegmentGroup::make_snapshot(const std::string& snapshot_path, return OLAP_SUCCESS; } +OLAPStatus SegmentGroup::copy_files_to_path(const std::string& dest_path, + std::vector* success_files) { + if (_empty) { + return OLAP_SUCCESS; + } + for (int segment_id = 0; segment_id < _num_segments; segment_id++) { + std::string dest_data_file = construct_data_file_path(dest_path, segment_id); + if (!check_dir_existed(dest_data_file)) { + std::string data_file_to_copy = construct_data_file_path(segment_id); + if (copy_file(data_file_to_copy, dest_data_file) != 0) { + LOG(WARNING) << "fail to copy data file. from=" << data_file_to_copy + << ", to=" << dest_data_file + << ", errno=" << Errno::no(); + return OLAP_ERR_OS_ERROR; + } + } + success_files->push_back(dest_data_file); + std::string dest_index_file = construct_index_file_path(dest_path, segment_id); + if (!check_dir_existed(dest_index_file)) { + std::string index_file_to_copy = construct_index_file_path(segment_id); + if (copy_file(index_file_to_copy, dest_index_file) != 0) { + LOG(WARNING) << "fail to copy index file. from=" << index_file_to_copy + << ", to=" << dest_index_file + << ", errno=" << Errno::no(); + return OLAP_ERR_OS_ERROR; + } + } + success_files->push_back(dest_index_file); + } + return OLAP_SUCCESS; +} + + OLAPStatus SegmentGroup::convert_from_old_files(const std::string& snapshot_path, std::vector* success_links) { if (_empty) { @@ -828,7 +861,7 @@ OLAPStatus SegmentGroup::remove_old_files(std::vector* links_to_rem return OLAP_SUCCESS; } -OLAPStatus SegmentGroup::copy_segments_to_path(const std::string& dest_path, int64_t rowset_id) { +OLAPStatus SegmentGroup::link_segments_to_path(const std::string& dest_path, int64_t rowset_id) { if (dest_path.empty()) { LOG(WARNING) << "dest path is empty, return error"; return OLAP_ERR_INPUT_PARAMETER_ERROR; diff --git a/be/src/olap/rowset/segment_group.h b/be/src/olap/rowset/segment_group.h index fa274d9ec65a54..5b68e9cc4475b5 100644 --- a/be/src/olap/rowset/segment_group.h +++ b/be/src/olap/rowset/segment_group.h @@ -261,8 +261,10 @@ class SegmentGroup { OLAPStatus make_snapshot(const std::string& snapshot_path, std::vector* success_links); + OLAPStatus copy_files_to_path(const std::string& dest_path, + std::vector* success_files); - OLAPStatus copy_segments_to_path(const std::string& dest_path, int64_t rowset_id); + OLAPStatus link_segments_to_path(const std::string& dest_path, int64_t rowset_id); private: diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index ff8fec866226b6..c409642d54a7bc 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -219,12 +219,16 @@ OLAPStatus EngineStorageMigrationTask::_copy_index_and_data_files( const string& schema_hash_path, const TabletSharedPtr& ref_tablet, std::vector& consistent_rowsets) { - // TODO(lcy). copy function should be implemented + std::vector success_files; + OLAPStatus status = OLAP_SUCCESS; for (auto& rs : consistent_rowsets) { - std::vector success_files; - RETURN_NOT_OK(rs->make_snapshot(schema_hash_path, &success_files)); + status = rs->copy_files_to_path(schema_hash_path, &success_files); + if (status != OLAP_SUCCESS) { + while (OLAP_SUCCESS != remove_all_dir(schema_hash_path)); + break; + } } - return OLAP_SUCCESS; + return status; } } // doris From 7bd2bd3b8bc328fd92e7498f0cfd64f9ef3d9d72 Mon Sep 17 00:00:00 2001 From: chaoyli Date: Thu, 13 Jun 2019 16:19:52 +0800 Subject: [PATCH 2/5] Add copy function to support storage migration task --- be/src/olap/rowset/segment_group.cpp | 36 +++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/be/src/olap/rowset/segment_group.cpp b/be/src/olap/rowset/segment_group.cpp index 8b3e3811bfa6e2..562c7a812e0ca7 100644 --- a/be/src/olap/rowset/segment_group.cpp +++ b/be/src/olap/rowset/segment_group.cpp @@ -730,25 +730,29 @@ OLAPStatus SegmentGroup::copy_files_to_path(const std::string& dest_path, } for (int segment_id = 0; segment_id < _num_segments; segment_id++) { std::string dest_data_file = construct_data_file_path(dest_path, segment_id); - if (!check_dir_existed(dest_data_file)) { - std::string data_file_to_copy = construct_data_file_path(segment_id); - if (copy_file(data_file_to_copy, dest_data_file) != 0) { - LOG(WARNING) << "fail to copy data file. from=" << data_file_to_copy - << ", to=" << dest_data_file - << ", errno=" << Errno::no(); - return OLAP_ERR_OS_ERROR; - } + if (check_dir_existed(dest_data_file)) { + LOG(WARNING) << "file already exists:" << dest_data_file; + return OLAP_ERR_FILE_ALREADY_EXIST; + } + std::string data_file_to_copy = construct_data_file_path(segment_id); + if (copy_file(data_file_to_copy, dest_data_file) != OLAP_SUCCESS) { + LOG(WARNING) << "fail to copy data file. from=" << data_file_to_copy + << ", to=" << dest_data_file + << ", errno=" << Errno::no(); + return OLAP_ERR_OS_ERROR; } success_files->push_back(dest_data_file); std::string dest_index_file = construct_index_file_path(dest_path, segment_id); - if (!check_dir_existed(dest_index_file)) { - std::string index_file_to_copy = construct_index_file_path(segment_id); - if (copy_file(index_file_to_copy, dest_index_file) != 0) { - LOG(WARNING) << "fail to copy index file. from=" << index_file_to_copy - << ", to=" << dest_index_file - << ", errno=" << Errno::no(); - return OLAP_ERR_OS_ERROR; - } + if (check_dir_existed(dest_index_file)) { + LOG(WARNING) << "file already exists:" << dest_index_file; + return OLAP_ERR_FILE_ALREADY_EXIST; + } + std::string index_file_to_copy = construct_index_file_path(segment_id); + if (copy_file(index_file_to_copy, dest_index_file) != OLAP_SUCCESS) { + LOG(WARNING) << "fail to copy index file. from=" << index_file_to_copy + << ", to=" << dest_index_file + << ", errno=" << Errno::no(); + return OLAP_ERR_OS_ERROR; } success_files->push_back(dest_index_file); } From 5b8e37f6559a915f0675691ef38bfe9d83fa9ec2 Mon Sep 17 00:00:00 2001 From: chaoyli Date: Thu, 13 Jun 2019 16:36:11 +0800 Subject: [PATCH 3/5] Add copy function to support storage migration task --- be/src/olap/task/engine_storage_migration_task.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index c409642d54a7bc..bb5417e2636d94 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -224,7 +224,10 @@ OLAPStatus EngineStorageMigrationTask::_copy_index_and_data_files( for (auto& rs : consistent_rowsets) { status = rs->copy_files_to_path(schema_hash_path, &success_files); if (status != OLAP_SUCCESS) { - while (OLAP_SUCCESS != remove_all_dir(schema_hash_path)); + if (remove_all_dir(schema_hash_path) != OLAP_SUCCESS) { + LOG(FATAL) << "remove storage migration path failed. " + << "schema_hash_path:" << schema_hash_path; + } break; } } From 1237c36cf4ac6d0f23a143ff9ad6084d3ea295d8 Mon Sep 17 00:00:00 2001 From: chaoyli Date: Thu, 13 Jun 2019 17:24:09 +0800 Subject: [PATCH 4/5] Add copy function to support storage migration task --- be/src/olap/olap_define.h | 1 + be/src/olap/rowset/alpha_rowset.h | 2 +- be/src/olap/tablet.h | 12 ------------ be/src/olap/task/engine_storage_migration_task.cpp | 14 ++++++++------ 4 files changed, 10 insertions(+), 19 deletions(-) diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h index dc5414a0062ecf..3f329cd9f03daa 100644 --- a/be/src/olap/olap_define.h +++ b/be/src/olap/olap_define.h @@ -344,6 +344,7 @@ enum OLAPStatus { OLAP_ERR_META_PUT = -3004, OLAP_ERR_META_ITERATOR = -3005, OLAP_ERR_META_DELETE = -3006, + OLAP_ERR_META_ALREADY_EXIST = -3007, // Rowset // [-3100, -3200) diff --git a/be/src/olap/rowset/alpha_rowset.h b/be/src/olap/rowset/alpha_rowset.h index b50acd233b3bb4..fc1c227b2acdbd 100644 --- a/be/src/olap/rowset/alpha_rowset.h +++ b/be/src/olap/rowset/alpha_rowset.h @@ -87,7 +87,7 @@ class AlphaRowset : public Rowset { OLAPStatus make_snapshot(const std::string& snapshot_path, std::vector* success_links) override; OLAPStatus copy_files_to_path(const std::string& dest_path, - std::vector* success_files); + std::vector* success_files) override; OLAPStatus convert_from_old_files(const std::string& snapshot_path, std::vector* success_files); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index a359f86474432c..6596f36efd7255 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -380,18 +380,6 @@ inline size_t Tablet::row_size() const { return _schema.row_size(); } -inline OLAPStatus Tablet::try_migration_rdlock() { - return _migration_lock.tryrdlock(); -} - -inline OLAPStatus Tablet::try_migration_wrlock() { - return _migration_lock.trywrlock(); -} - -inline void Tablet::release_migration_lock() { - _migration_lock.unlock(); -} - } #endif // DORIS_BE_SRC_OLAP_TABLET_H diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index bb5417e2636d94..90d70c1f472cd4 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -118,6 +118,14 @@ OLAPStatus EngineStorageMigrationTask::_storage_medium_migrate( << "schema_hash_path=" << schema_hash_path; remove_all_dir(schema_hash_path); } + TabletMetaSharedPtr new_tablet_meta(new(std::nothrow) TabletMeta()); + TabletMetaManager::get_header(stores[0], tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta); + if (new_tablet_meta != nullptr) { + LOG(WARNING) << "tablet_meta already exists. " + << "data_dir:" << stores[0]->path() + << "tablet:" << tablet->full_name(); + return OLAP_ERR_META_ALREADY_EXIST; + } create_dirs(schema_hash_path); // migrate all index and data files but header file @@ -127,12 +135,6 @@ OLAPStatus EngineStorageMigrationTask::_storage_medium_migrate( break; } - // generate new header file from the old - TabletMetaSharedPtr new_tablet_meta(new(std::nothrow) TabletMeta()); - if (new_tablet_meta == nullptr) { - LOG(WARNING) << "new olap header failed"; - return OLAP_ERR_BUFFER_OVERFLOW; - } res = _generate_new_header(stores[0], shard, tablet, consistent_rowsets, new_tablet_meta); if (res != OLAP_SUCCESS) { LOG(WARNING) << "fail to generate new header file from the old. res=" << res; From 2a45914db9861011b1181719dcbbaf0c54cec2f1 Mon Sep 17 00:00:00 2001 From: chaoyli Date: Thu, 13 Jun 2019 19:03:08 +0800 Subject: [PATCH 5/5] Add copy function to support storage migration task --- be/src/olap/task/engine_storage_migration_task.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index 90d70c1f472cd4..9d7e4b3d7b0e2c 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -119,8 +119,8 @@ OLAPStatus EngineStorageMigrationTask::_storage_medium_migrate( remove_all_dir(schema_hash_path); } TabletMetaSharedPtr new_tablet_meta(new(std::nothrow) TabletMeta()); - TabletMetaManager::get_header(stores[0], tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta); - if (new_tablet_meta != nullptr) { + res = TabletMetaManager::get_header(stores[0], tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta); + if (res != OLAP_ERR_META_KEY_NOT_FOUND) { LOG(WARNING) << "tablet_meta already exists. " << "data_dir:" << stores[0]->path() << "tablet:" << tablet->full_name();