From 3b46c95749f37eefb0d3f505565fc0ea24ce9c6e Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Thu, 12 Dec 2024 18:56:21 +0800 Subject: [PATCH 01/15] [feature](cooldown)backup cooldown data --- be/src/io/fs/s3_file_system.h | 5 + be/src/olap/olap_define.h | 1 + be/src/olap/olap_server.cpp | 2 +- be/src/olap/rowset/beta_rowset.cpp | 78 +++++- be/src/olap/rowset/beta_rowset.h | 5 +- be/src/olap/rowset/rowset.h | 8 +- be/src/olap/rowset/rowset_meta.cpp | 4 + be/src/olap/rowset/rowset_meta.h | 2 + be/src/olap/snapshot_manager.cpp | 65 ++++- be/src/olap/tablet.cpp | 12 + be/src/olap/tablet.h | 2 + be/src/olap/tablet_meta.cpp | 9 + be/src/olap/tablet_meta.h | 2 + be/src/runtime/snapshot_loader.cpp | 143 +++++++++++ .../org/apache/doris/backup/BackupJob.java | 41 ++- .../apache/doris/backup/BackupJobInfo.java | 37 +++ .../org/apache/doris/backup/BackupMeta.java | 46 +++- .../org/apache/doris/backup/RestoreJob.java | 128 +++++++++- .../org/apache/doris/catalog/ResourceMgr.java | 7 + .../org/apache/doris/catalog/S3Resource.java | 37 +++ .../java/org/apache/doris/policy/Policy.java | 4 + .../org/apache/doris/policy/PolicyMgr.java | 7 +- .../apache/doris/policy/StoragePolicy.java | 22 +- .../doris/service/FrontendServiceImpl.java | 15 ++ .../doris/backup/BackupHandlerTest.java | 4 +- .../apache/doris/backup/RestoreJobTest.java | 4 +- .../test_backup_restore_cold_data.groovy | 238 ++++++++++++++++++ 27 files changed, 907 insertions(+), 21 deletions(-) create mode 100644 regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h index f6efa5053324ff..ed671425468f1a 100644 --- a/be/src/io/fs/s3_file_system.h +++ b/be/src/io/fs/s3_file_system.h @@ -120,8 +120,13 @@ class S3FileSystem final : public RemoteFileSystem { // so no need to concat with prefix abs_path = path; } else { + std::string path_str = _root_path.string(); + if (!path_str.empty() && path_str[0] == '/') { + path_str.erase(0, 1); // remove first '/' + } // path with no schema abs_path = _prefix / path; + return std::filesystem::path(path_str) / path; } return Status::OK(); } diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h index 5131c51ca01c20..bb767f9a9f6e69 100644 --- a/be/src/olap/olap_define.h +++ b/be/src/olap/olap_define.h @@ -99,6 +99,7 @@ static const std::string INCREMENTAL_DELTA_PREFIX = "incremental_delta"; static const std::string CLONE_PREFIX = "clone"; static const std::string SPILL_DIR_PREFIX = "spill"; static const std::string SPILL_GC_DIR_PREFIX = "spill_gc"; +static const std::string REMOTE_FILE_INFO = "remote_file_info"; static inline std::string local_segment_path(std::string_view tablet_path, std::string_view rowset_id, int64_t seg_id) { diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 90d0883984e78b..8884b1b1f926b2 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -1314,7 +1314,7 @@ void StorageEngine::do_remove_unused_remote_files() { } cooldown_meta_id = t->tablet_meta()->cooldown_meta_id(); } - auto [cooldown_replica_id, cooldown_term] = t->cooldown_conf(); + auto [cooldown_term, cooldown_replica_id] = t->cooldown_conf(); if (cooldown_replica_id != t->replica_id()) { return; } diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index cd52deed0c8a4d..4196a79305ae06 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -432,13 +432,89 @@ Status BetaRowset::copy_files_to(const std::string& dir, const RowsetId& new_row return Status::OK(); } +Status BetaRowset::download(io::RemoteFileSystem* fs, const std::string& dir) { + if (is_local()) { + DCHECK(false) << _rowset_meta->tablet_id() << ' ' << rowset_id(); + return Status::InternalError("should be remote rowset. tablet_id={} rowset_id={}", + _rowset_meta->tablet_id(), rowset_id().to_string()); + } + + if (num_segments() < 1) { + return Status::OK(); + } + + Status status; + std::vector linked_success_files; + Defer remove_linked_files {[&]() { // clear download files if errors happen + if (!status.ok()) { + LOG(WARNING) << "will delete download success files due to error " << status; + std::vector paths; + for (auto& file : linked_success_files) { + paths.emplace_back(file); + LOG(WARNING) << "will delete download success file " << file << " due to error"; + } + static_cast(fs->batch_delete(paths)); + LOG(WARNING) << "done delete download success files due to error " << status; + } + }}; + + for (int i = 0; i < num_segments(); ++i) { + // Note: Here we use relative path for remote. + auto remote_seg_path = + remote_segment_path(_rowset_meta->tablet_id(), rowset_id().to_string(), i); + + auto local_seg_path = segment_file_path(dir, rowset_id(), i); + + RETURN_IF_ERROR(fs->download(remote_seg_path, local_seg_path)); + + linked_success_files.push_back(local_seg_path); + + if (_schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1) { + if (_schema->has_inverted_index()) { + std::string inverted_index_src_file = + InvertedIndexDescriptor::get_index_file_name(remote_seg_path); + + std::string inverted_index_dst_file_path = + InvertedIndexDescriptor::get_index_file_name(local_seg_path); + + RETURN_IF_ERROR( + fs->download(inverted_index_src_file, inverted_index_dst_file_path)); + + linked_success_files.push_back(inverted_index_dst_file_path); + } + } else { + for (const auto& index : _schema->indexes()) { + if (index.index_type() != IndexType::INVERTED) { + continue; + } + + auto index_id = index.index_id(); + std::string inverted_index_src_file = InvertedIndexDescriptor::get_index_file_name( + remote_seg_path, index_id, index.get_index_suffix()); + + std::string inverted_index_dst_file_path = + InvertedIndexDescriptor::get_index_file_name(local_seg_path, index_id, + index.get_index_suffix()); + + RETURN_IF_ERROR( + fs->download(inverted_index_src_file, inverted_index_dst_file_path)); + + linked_success_files.push_back(inverted_index_dst_file_path); + LOG(INFO) << "success to download. from=" << inverted_index_src_file << ", " + << "to=" << inverted_index_dst_file_path; + } + } + } + + return Status::OK(); +} + Status BetaRowset::upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) { if (!is_local()) { DCHECK(false) << _rowset_meta->tablet_id() << ' ' << rowset_id(); return Status::InternalError("should be local rowset. tablet_id={} rowset_id={}", _rowset_meta->tablet_id(), rowset_id().to_string()); } - if (num_segments() < 1) { return Status::OK(); } diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 52d5ac5c8a8742..dd8d536a29bd87 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -62,7 +62,10 @@ class BetaRowset final : public Rowset { Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) override; - Status upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) override; + + Status download(io::RemoteFileSystem* fs, const std::string& dir) override; + + Status upload_to(io::RemoteFileSystem* dest_fs, const RowsetId& new_rowset_id) override; // only applicable to alpha rowset, no op here Status remove_old_files(std::vector* files_to_remove) override { diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index 98d88ba19f2068..cdb370c809a1c9 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -213,7 +213,13 @@ class Rowset : public std::enable_shared_from_this, public MetadataAdder // copy all files to `dir` virtual Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) = 0; - virtual Status upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) = 0; + virtual Status download(io::RemoteFileSystem* fs, const std::string& dir) { + return Status::OK(); + } + + virtual Status upload_to(io::RemoteFileSystem* dest_fs, const RowsetId& new_rowset_id) { + return Status::OK(); + } virtual Status remove_old_files(std::vector* files_to_remove) = 0; diff --git a/be/src/olap/rowset/rowset_meta.cpp b/be/src/olap/rowset/rowset_meta.cpp index 6bed5e800ede4d..034aab25ea569d 100644 --- a/be/src/olap/rowset/rowset_meta.cpp +++ b/be/src/olap/rowset/rowset_meta.cpp @@ -133,6 +133,10 @@ bool RowsetMeta::has_variant_type_in_schema() const { return _schema && _schema->num_variant_columns() > 0; } +void RowsetMeta::clear_resource_id() { + _rowset_meta_pb.clear_resource_id(); +} + void RowsetMeta::to_rowset_pb(RowsetMetaPB* rs_meta_pb, bool skip_schema) const { *rs_meta_pb = _rowset_meta_pb; if (_schema) [[likely]] { diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index 46121aeae2be6d..592a503663b042 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -60,6 +60,8 @@ class RowsetMeta : public MetadataAdder { void set_remote_storage_resource(StorageResource resource); + void clear_resource_id(); + const std::string& resource_id() const { return _rowset_meta_pb.resource_id(); } bool is_local() const { return !_rowset_meta_pb.has_resource_id(); } diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index 67205835b53947..c8adff009a2d9f 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -39,6 +39,7 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "olap/data_dir.h" #include "olap/olap_common.h" @@ -461,6 +462,7 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet "missed version is a cooldowned rowset, must make full " "snapshot. missed_version={}, tablet_id={}", missed_version, ref_tablet->tablet_id()); + //todozy break; } consistent_rowsets.push_back(rowset); @@ -492,8 +494,8 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet LOG(WARNING) << "currently not support backup tablet with cooldowned remote " "data. tablet=" << request.tablet_id; - return Status::NotSupported( - "currently not support backup tablet with cooldowned remote data"); + // return Status::NotSupported( + // "currently not support backup tablet with cooldowned remote data"); } /// not all missing versions are found, fall back to full snapshot. res = Status::OK(); // reset res @@ -566,6 +568,10 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet } std::vector rs_metas; + RowsetMetaSharedPtr rsm; + bool have_remote_file = false; + io::FileWriterPtr file_writer; + for (auto& rs : consistent_rowsets) { if (rs->is_local()) { // local rowset @@ -573,12 +579,56 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet if (!res.ok()) { break; } + rsm = rs->rowset_meta(); + } else { + std::string rowset_meta_str; + RowsetMetaPB rs_meta_pb; + rs->rowset_meta()->to_rowset_pb(&rs_meta_pb); + rs_meta_pb.SerializeToString(&rowset_meta_str); + + RowsetMetaSharedPtr rowset_meta(new RowsetMeta()); + rowset_meta->init(rowset_meta_str); + + rsm = rowset_meta; + + // save_remote_file info + // tableid|storage_policy_id| + // rowset_id|num_segments|has_inverted_index| + // ...... + // rowset_id|num_segments|has_inverted_index + { + // write file + std::string delimeter = "|"; + + if (!have_remote_file) { + auto romote_file_info = + fmt::format("{}/{}", schema_full_path, REMOTE_FILE_INFO); + RETURN_IF_ERROR(io::global_local_filesystem()->create_file(romote_file_info, + &file_writer)); + RETURN_IF_ERROR(file_writer->append( + std::to_string(rs->rowset_meta()->tablet_id()))); + RETURN_IF_ERROR(file_writer->append(delimeter)); + RETURN_IF_ERROR(file_writer->append( + std::to_string(ref_tablet->tablet_meta()->storage_policy_id()))); + have_remote_file = true; + } + RETURN_IF_ERROR(file_writer->append(delimeter)); + RETURN_IF_ERROR(file_writer->append(rs->rowset_id().to_string())); + RETURN_IF_ERROR(file_writer->append(delimeter)); + RETURN_IF_ERROR(file_writer->append(std::to_string(rs->num_segments()))); + RETURN_IF_ERROR(file_writer->append(delimeter)); + RETURN_IF_ERROR(file_writer->append( + std::to_string(rs->tablet_schema()->has_inverted_index()))); + } } - rs_metas.push_back(rs->rowset_meta()); + rs_metas.push_back(rsm); VLOG_NOTICE << "add rowset meta to clone list. " - << " start version " << rs->rowset_meta()->start_version() - << " end version " << rs->rowset_meta()->end_version() << " empty " - << rs->rowset_meta()->empty(); + << " start version " << rsm->start_version() << " end version " + << rsm->end_version() << " empty " << rsm->empty(); + } + + if (have_remote_file) { + RETURN_IF_ERROR(file_writer->close()); } if (!res.ok()) { LOG(WARNING) << "fail to create hard link. path=" << snapshot_id_path @@ -596,6 +646,9 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet new_tablet_meta->revise_delete_bitmap_unlocked(delete_bitmap_snapshot); } + //clear cooldown meta + new_tablet_meta->revise_clear_resource_id(); + if (snapshot_version == g_Types_constants.TSNAPSHOT_REQ_VERSION2) { res = new_tablet_meta->save(header_path); if (res.ok() && request.__isset.is_copy_tablet_task && request.is_copy_tablet_task) { diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index a1a56507ffc67a..d75d1affadfebf 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2025,6 +2025,18 @@ Status Tablet::cooldown(RowsetSharedPtr rowset) { return Status::OK(); } +Status Tablet::download(RowsetSharedPtr rowset, const std::string& dir) { + std::shared_ptr dest_fs; + RETURN_IF_ERROR(get_remote_file_system(storage_policy_id(), &dest_fs)); + Status st; + + if (st = rowset->download(dest_fs.get(), dir); !st.ok()) { + return st; + } + + return Status::OK(); +} + // hold SHARED `cooldown_conf_lock` Status Tablet::_cooldown_data(RowsetSharedPtr rowset) { DCHECK(_cooldown_conf.cooldown_replica_id == replica_id()); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 40b911d6391b9b..f280688a7f63ff 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -378,6 +378,8 @@ class Tablet final : public BaseTablet { // Cooldown to remote fs. Status cooldown(RowsetSharedPtr rowset = nullptr); + Status download(RowsetSharedPtr rowset, const std::string& dir); + RowsetSharedPtr pick_cooldown_rowset(); RowsetSharedPtr need_cooldown(int64_t* cooldown_timestamp, size_t* file_size); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 43b0d5d8bd0ae0..5788205f7edac7 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -886,6 +886,15 @@ void TabletMeta::revise_rs_metas(std::vector&& rs_metas) { _stale_rs_metas.clear(); } +void TabletMeta::revise_clear_resource_id() { + for (auto rs : _rs_metas) { + rs->clear_resource_id(); + } + for (auto rs : _stale_rs_metas) { + rs->clear_resource_id(); + } +} + // This method should call after revise_rs_metas, since new rs_metas might be a subset // of original tablet, we should revise the delete_bitmap according to current rowset. // diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 25f6bcd569be43..303257ffb34036 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -198,6 +198,8 @@ class TabletMeta : public MetadataAdder { void modify_rs_metas(const std::vector& to_add, const std::vector& to_delete, bool same_version = false); + + void revise_clear_resource_id(); void revise_rs_metas(std::vector&& rs_metas); void revise_delete_bitmap_unlocked(const DeleteBitmap& delete_bitmap); diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index 784904c78a3fb1..25c1aef1ad2c1b 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -38,6 +38,7 @@ #include "gutil/strings/split.h" #include "http/http_client.h" #include "io/fs/broker_file_system.h" +#include "io/fs/file_reader.h" #include "io/fs/file_system.h" #include "io/fs/hdfs_file_system.h" #include "io/fs/local_file_system.h" @@ -48,6 +49,7 @@ #include "olap/data_dir.h" #include "olap/snapshot_manager.h" #include "olap/storage_engine.h" +#include "olap/storage_policy.h" #include "olap/tablet.h" #include "olap/tablet_manager.h" #include "runtime/client_cache.h" @@ -120,6 +122,137 @@ Status SnapshotLoader::init(TStorageBackendType::type type, const std::string& l SnapshotLoader::~SnapshotLoader() = default; +static Status list_segment_inverted_index_file(io::RemoteFileSystem* cold_fs, + const std::string& dir, const std::string& rowset, + std::vector* remote_files) { + bool exists = true; + std::vector files; + RETURN_IF_ERROR(cold_fs->list(dir, true, &files, &exists)); + for (auto& tmp_file : files) { + io::Path path(tmp_file.file_name); + std::string file_name = path.filename(); + + if (file_name.substr(0, rowset.length()).compare(rowset) != 0 || + !_end_with(file_name, ".idx")) { + continue; + } + remote_files->push_back(file_name); + } + + return Status::OK(); +} + +static Status download_and_upload_one_file(io::RemoteFileSystem& dest_fs, + io::RemoteFileSystem* cold_fs, + const std::string& remote_seg_path, + const std::string& local_seg_path, + const std::string& dest_seg_path) { + RETURN_IF_ERROR(cold_fs->download(remote_seg_path, local_seg_path)); + + // calc md5sum of localfile + std::string md5sum; + RETURN_IF_ERROR(io::global_local_filesystem()->md5sum(local_seg_path, &md5sum)); + + RETURN_IF_ERROR(upload_with_checksum(dest_fs, local_seg_path, dest_seg_path, md5sum)); + + //delete local file + RETURN_IF_ERROR(io::global_local_filesystem()->delete_file(local_seg_path)); + + return Status::OK(); +} + +static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet_id, + const std::string& local_path, const std::string& dest_path, + io::RemoteFileSystem* cold_fs, const std::string& rowset, + int segments, int have_inverted_index) { + Status res = Status::OK(); + + for (int i = 0; i < segments; i++) { + std::string remote_seg_path = + fmt::format("{}/{}_{}.dat", remote_tablet_path(tablet_id), rowset, i); + std::string local_seg_path = fmt::format("{}/{}_{}.dat", local_path, rowset, i); + std::string dest_seg_path = fmt::format("{}/{}_{}.dat", dest_path, rowset, i); + + RETURN_IF_ERROR(download_and_upload_one_file(dest_fs, cold_fs, remote_seg_path, + local_seg_path, dest_seg_path)); + } + + if (!have_inverted_index) { + return res; + } + + std::vector remote_index_files; + RETURN_IF_ERROR(list_segment_inverted_index_file(cold_fs, remote_tablet_path(tablet_id), rowset, + &remote_index_files)); + + for (auto& index_file : remote_index_files) { + std::string remote_index_path = + fmt::format("{}/{}", remote_tablet_path(tablet_id), index_file); + std::string local_seg_path = fmt::format("{}/{}", local_path, index_file); + std::string dest_seg_path = fmt::format("{}/{}", dest_path, index_file); + + RETURN_IF_ERROR(download_and_upload_one_file(dest_fs, cold_fs, remote_index_path, + local_seg_path, dest_seg_path)); + } + return res; +} + +static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_id, + const std::string& local_path, const std::string& dest_path, + const std::string& remote_file) { + io::FileReaderSPtr file_reader; + Status res = Status::OK(); + + std::string full_remote_path = local_path + '/' + remote_file; + RETURN_IF_ERROR(io::global_local_filesystem()->open_file(full_remote_path, &file_reader)); + size_t bytes_read = 0; + char* buff = (char*)malloc(file_reader->size() + 1); + RETURN_IF_ERROR(file_reader->read_at(0, {buff, file_reader->size()}, &bytes_read)); + string str(buff, file_reader->size()); + size_t start = 0; + string delimiter = "|"; + size_t end = str.find(delimiter); + int64_t tablet_id_tmp = std::stol(str.substr(start, end - start)); + start = end + delimiter.length(); + + if (tablet_id_tmp != tablet_id) { + return Status::InternalError("Invalid tablet {}", tablet_id_tmp); + } + + end = str.find(delimiter, start); // + int64_t storage_policy_id = std::stol(str.substr(start, end - start)); + start = end + delimiter.length(); + + string rowset_id; + int segments; + int have_inverted_index; + + std::shared_ptr colddata_fs; + RETURN_IF_ERROR(get_remote_file_system(storage_policy_id, &colddata_fs)); + + while (end != std::string::npos) { + end = str.find(delimiter, start); // + rowset_id = str.substr(start, end - start); + start = end + delimiter.length(); + + end = str.find(delimiter, start); + segments = std::stoi(str.substr(start, end - start)); + start = end + delimiter.length(); + + end = str.find(delimiter, start); + have_inverted_index = std::stoi(str.substr(start, end - start)); + start = end + delimiter.length(); + + if (segments > 0) { + RETURN_IF_ERROR(upload_remote_rowset(dest_fs, tablet_id, local_path, dest_path, + colddata_fs.get(), rowset_id, segments, + have_inverted_index)); + } + } + + return res; +} + Status SnapshotLoader::upload(const std::map& src_to_dest_path, std::map>* tablet_files) { if (!_remote_fs) { @@ -169,6 +302,11 @@ Status SnapshotLoader::upload(const std::map& src_to_d RETURN_IF_ERROR(_report_every(10, &report_counter, finished_num, total_num, TTaskType::type::UPLOAD)); + const std::string& local_file = *it; + if (local_file.compare("remote_file_info") == 0) { + RETURN_IF_ERROR(upload_remote_file(*_remote_fs, tablet_id, src_path, dest_path, + local_file)); + } // calc md5sum of localfile std::string md5sum; RETURN_IF_ERROR( @@ -286,12 +424,17 @@ Status SnapshotLoader::download(const std::map& src_to const FileStat& file_stat = iter.second; auto find = std::find(local_files.begin(), local_files.end(), remote_file); if (find == local_files.end()) { + if (remote_file.compare(REMOTE_FILE_INFO) == 0) { + continue; + } // remote file does not exist in local, download it need_download = true; } else { if (_end_with(remote_file, ".hdr")) { // this is a header file, download it. need_download = true; + } else if (remote_file.compare(REMOTE_FILE_INFO) == 0) { + continue; } else { // check checksum std::string local_md5sum; diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index de12670807f20e..4189851a7c0c91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -29,6 +29,7 @@ import org.apache.doris.catalog.OdbcTable; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.Table; @@ -43,6 +44,7 @@ import org.apache.doris.persist.BarrierLog; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.policy.StoragePolicy; import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskExecutor; @@ -62,6 +64,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -495,6 +498,8 @@ private void prepareAndSendSnapshotTask() { List copiedTables = Lists.newArrayList(); List copiedResources = Lists.newArrayList(); AgentBatchTask batchTask = new AgentBatchTask(Config.backup_restore_batch_task_num_per_rpc); + List copiedStoragePolicys = Lists.newArrayList(); + AgentBatchTask batchTask = new AgentBatchTask(Config.backup_restore_batch_task_num_per_rpc); for (TableRef tableRef : tableRefs) { String tblName = tableRef.getName().getTbl(); Table tbl = db.getTableNullable(tblName); @@ -511,9 +516,15 @@ private void prepareAndSendSnapshotTask() { OlapTable olapTable = (OlapTable) tbl; checkOlapTable(olapTable, tableRef); if (getContent() == BackupContent.ALL) { - prepareSnapshotTaskForOlapTableWithoutLock(db, (OlapTable) tbl, tableRef, batchTask); + if (!prepareSnapshotTaskForOlapTableWithoutLock( + db, (OlapTable) tbl, tableRef, batchTask).ok()) { + return; + } + } + if (!prepareBackupMetaForOlapTableWithoutLock(tableRef, olapTable, copiedTables, + copiedStoragePolicys).ok()) { + return; } - prepareBackupMetaForOlapTableWithoutLock(tableRef, olapTable, copiedTables); break; case VIEW: prepareBackupMetaForViewWithoutLock((View) tbl, copiedTables); @@ -547,7 +558,7 @@ private void prepareAndSendSnapshotTask() { return; } - backupMeta = new BackupMeta(copiedTables, copiedResources); + backupMeta = new BackupMeta(copiedTables, copiedResources, copiedStoragePolicys); // send tasks for (AgentTask task : batchTask.getAllTasks()) { @@ -665,8 +676,9 @@ private void checkResourceForOdbcTable(OdbcTable odbcTable) { } } - private void prepareBackupMetaForOlapTableWithoutLock(TableRef tableRef, OlapTable olapTable, - List
copiedTables) { + private Status prepareBackupMetaForOlapTableWithoutLock(TableRef tableRef, OlapTable olapTable, + List
copiedTables, + List copiedStoragePolicys) { // only copy visible indexes List reservedPartitions = tableRef.getPartitionNames() == null ? null : tableRef.getPartitionNames().getPartitionNames(); @@ -678,6 +690,25 @@ private void prepareBackupMetaForOlapTableWithoutLock(TableRef tableRef, OlapTab removeUnsupportProperties(copiedTbl); copiedTables.add(copiedTbl); + + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + // classify a table's all partitions by storage policy + for (Long partitionId : olapTable.getPartitionIds()) { + String policyName = partitionInfo.getDataProperty(partitionId).getStoragePolicy(); + if (StringUtils.isEmpty(policyName)) { + continue; + } + + StoragePolicy checkedPolicyCondition = StoragePolicy.ofCheck(policyName); + StoragePolicy storagePolicy = (StoragePolicy) Env.getCurrentEnv().getPolicyMgr() + .getPolicy(checkedPolicyCondition); + + if (storagePolicy != null && !copiedStoragePolicys.contains(storagePolicy)) { + copiedStoragePolicys.add(storagePolicy); + } + } + + return Status.OK; } private void prepareBackupMetaForViewWithoutLock(View view, List
copiedTables) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java index 554a21c44080f7..09b4c8fdf08c82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java @@ -29,6 +29,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Resource; +import org.apache.doris.catalog.S3Resource; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Tablet; @@ -41,6 +42,7 @@ import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.policy.StoragePolicy; import org.apache.doris.thrift.TNetworkAddress; import com.google.common.base.Joiner; @@ -335,6 +337,10 @@ public static class BriefBackupJobInfo { public List odbcTableList = Lists.newArrayList(); @SerializedName("odbc_resource_list") public List odbcResourceList = Lists.newArrayList(); + @SerializedName("s3_resource_list") + public List s3ResourceList = Lists.newArrayList(); + @SerializedName("storage_policy_list") + public List storagePolicyList = Lists.newArrayList(); public static BriefBackupJobInfo fromBackupJobInfo(BackupJobInfo backupJobInfo) { BriefBackupJobInfo briefBackupJobInfo = new BriefBackupJobInfo(); @@ -352,6 +358,8 @@ public static BriefBackupJobInfo fromBackupJobInfo(BackupJobInfo backupJobInfo) briefBackupJobInfo.viewList = backupJobInfo.newBackupObjects.views; briefBackupJobInfo.odbcTableList = backupJobInfo.newBackupObjects.odbcTables; briefBackupJobInfo.odbcResourceList = backupJobInfo.newBackupObjects.odbcResources; + briefBackupJobInfo.s3ResourceList = backupJobInfo.newBackupObjects.s3Resources; + briefBackupJobInfo.storagePolicyList = backupJobInfo.newBackupObjects.storagePolicies; return briefBackupJobInfo; } } @@ -370,6 +378,10 @@ public static class NewBackupObjects { public List odbcTables = Lists.newArrayList(); @SerializedName("odbc_resources") public List odbcResources = Lists.newArrayList(); + @SerializedName("s3_resources") + public List s3Resources = Lists.newArrayList(); + @SerializedName("storage_policy") + public List storagePolicies = Lists.newArrayList(); } public static class BackupOlapTableInfo { @@ -488,6 +500,16 @@ public static class BackupOdbcResourceInfo { public String name; } + public static class BackupS3ResourceInfo { + @SerializedName("name") + public String name; + } + + public static class BackupStoragePolicyInfo { + @SerializedName("name") + public String name; + } + // eg: __db_10001/__tbl_10002/__part_10003/__idx_10002/__10004 public String getFilePath(String db, String tbl, String part, String idx, long tabletId) { if (!db.equalsIgnoreCase(dbName)) { @@ -687,6 +709,21 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db backupOdbcResourceInfo.name = odbcCatalogResource.getName(); jobInfo.newBackupObjects.odbcResources.add(backupOdbcResourceInfo); } + + if (resource instanceof S3Resource) { + S3Resource s3Resource = (S3Resource) resource; + BackupS3ResourceInfo backupS3ResourceInfo = new BackupS3ResourceInfo(); + backupS3ResourceInfo.name = s3Resource.getName(); + jobInfo.newBackupObjects.s3Resources.add(backupS3ResourceInfo); + } + } + + // storage policies + Collection storagePolicies = backupMeta.getStoragePolicyNameMap().values(); + for (StoragePolicy storagePolicy : storagePolicies) { + BackupStoragePolicyInfo backupStoragePolicyInfo = new BackupStoragePolicyInfo(); + backupStoragePolicyInfo.name = storagePolicy.getName(); + jobInfo.newBackupObjects.storagePolicies.add(backupStoragePolicyInfo); } return jobInfo; diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java index 0f1a043bdada3b..294b4fb3ff9495 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java @@ -26,6 +26,7 @@ import org.apache.doris.meta.MetaContext; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.policy.StoragePolicy; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -52,11 +53,14 @@ public class BackupMeta implements Writable, GsonPostProcessable { // resource name -> resource @SerializedName(value = "resourceNameMap") private Map resourceNameMap = Maps.newHashMap(); + // storagePolicy name -> resource + @SerializedName(value = "storagePolicyNameMap") + private Map storagePolicyNameMap = Maps.newHashMap(); private BackupMeta() { } - public BackupMeta(List
tables, List resources) { + public BackupMeta(List
tables, List resources, List storagePolicys) { for (Table table : tables) { tblNameMap.put(table.getName(), table); tblIdMap.put(table.getId(), table); @@ -64,6 +68,21 @@ public BackupMeta(List
tables, List resources) { for (Resource resource : resources) { resourceNameMap.put(resource.getName(), resource); } + + for (StoragePolicy policy : storagePolicys) { + storagePolicyNameMap.put(policy.getName(), policy); + + if (resourceNameMap.get(policy.getStorageResource()) != null) { + continue; + } + Resource resource = Env.getCurrentEnv().getResourceMgr() + .getResource(policy.getStorageResource()); + Resource copiedResource = resource.clone(); + if (copiedResource == null) { + continue; + } + resourceNameMap.put(policy.getStorageResource(), copiedResource); + } } public Map getTables() { @@ -74,6 +93,10 @@ public Map getResourceNameMap() { return resourceNameMap; } + public Map getStoragePolicyNameMap() { + return storagePolicyNameMap; + } + public Table getTable(String tblName) { return tblNameMap.get(tblName); } @@ -82,6 +105,10 @@ public Resource getResource(String resourceName) { return resourceNameMap.get(resourceName); } + public StoragePolicy getStoragePolicy(String policyName) { + return storagePolicyNameMap.get(policyName); + } + public Table getTable(Long tblId) { return tblIdMap.get(tblId); } @@ -130,6 +157,18 @@ public static BackupMeta read(DataInput in) throws IOException { @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); + out.writeInt(tblNameMap.size()); + for (Table table : tblNameMap.values()) { + table.write(out); + } + out.writeInt(resourceNameMap.size()); + for (Resource resource : resourceNameMap.values()) { + resource.write(out); + } + out.writeInt(storagePolicyNameMap.size()); + for (StoragePolicy storagePolicy : storagePolicyNameMap.values()) { + storagePolicy.write(out); + } } @Deprecated @@ -145,6 +184,11 @@ public void readFields(DataInput in) throws IOException { Resource resource = Resource.read(in); resourceNameMap.put(resource.getName(), resource); } + size = in.readInt(); + for (int i = 0; i < size; i++) { + StoragePolicy policy = StoragePolicy.read(in); + storagePolicyNameMap.put(policy.getName(), policy); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 6dfd02b3a42648..a3606cfd09ab74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -49,6 +49,7 @@ import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.ResourceMgr; +import org.apache.doris.catalog.S3Resource; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Tablet; @@ -68,8 +69,15 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.property.S3ClientBEProperties; +<<<<<<< HEAD import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; +======= +import org.apache.doris.policy.Policy; +import org.apache.doris.policy.PolicyMgr; +import org.apache.doris.policy.PolicyTypeEnum; +import org.apache.doris.policy.StoragePolicy; +>>>>>>> 9bd9133568... [feature](cooldown)backup cooldown data import org.apache.doris.resource.Tag; import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTask; @@ -110,6 +118,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -182,6 +191,7 @@ public enum RestoreJobState { private List
restoredTbls = Lists.newArrayList(); @SerializedName("rr") private List restoredResources = Lists.newArrayList(); + private List storagePolicies = Lists.newArrayList(); // save all restored partitions' version info which are already exist in catalog // table id -> partition id -> (version, version hash) @@ -665,6 +675,19 @@ private void checkAndPrepareMeta() { } } + for (BackupJobInfo.BackupS3ResourceInfo backupS3ResourceInfo : jobInfo.newBackupObjects.s3Resources) { + Resource resource = Env.getCurrentEnv().getResourceMgr().getResource(backupS3ResourceInfo.name); + if (resource == null) { + continue; + } + if (resource.getType() != Resource.ResourceType.S3) { + status = new Status(ErrCode.COMMON_ERROR, + "The local resource " + resource.getName() + + " with the same name but a different type of backup meta."); + return; + } + } + // the new tablets -> { local tablet, schema hash, storage medium }, used in atomic restore. Map tabletBases = new HashMap<>(); @@ -957,6 +980,12 @@ private void checkAndPrepareMeta() { if (!status.ok()) { return; } + // check and restore storage policies + checkAndRestoreStoragePolicies(); + if (!status.ok()) { + return; + } + if (LOG.isDebugEnabled()) { LOG.debug("finished to restore resources. {}", this.jobId); } @@ -1253,7 +1282,7 @@ private void checkAndRestoreResources() { } else { try { // restore resource - resourceMgr.createResource(remoteOdbcResource, false); + resourceMgr.createResource(remoteOdbcResource); } catch (DdlException e) { status = new Status(ErrCode.COMMON_ERROR, e.getMessage()); return; @@ -1261,6 +1290,66 @@ private void checkAndRestoreResources() { restoredResources.add(remoteOdbcResource); } } + + for (BackupJobInfo.BackupS3ResourceInfo backupS3ResourceInfo : jobInfo.newBackupObjects.s3Resources) { + String backupResourceName = backupS3ResourceInfo.name; + Resource localResource = resourceMgr.getResource(backupResourceName); + S3Resource remoteS3Resource = (S3Resource) backupMeta.getResource(backupResourceName); + if (localResource != null) { + if (localResource.getType() != Resource.ResourceType.S3) { + status = new Status(ErrCode.COMMON_ERROR, "The type of local resource " + + backupResourceName + " is not same as restored resource"); + return; + } + S3Resource localS3Resource = (S3Resource) localResource; + if (localS3Resource.getSignature(BackupHandler.SIGNATURE_VERSION) + != remoteS3Resource.getSignature(BackupHandler.SIGNATURE_VERSION)) { + status = new Status(ErrCode.COMMON_ERROR, "S3 resource " + + jobInfo.getAliasByOriginNameIfSet(backupResourceName) + + " already exist but with different properties"); + return; + } + } else { + try { + // restore resource + resourceMgr.createResource(remoteS3Resource); + } catch (DdlException e) { + status = new Status(ErrCode.COMMON_ERROR, e.getMessage()); + return; + } + restoredResources.add(remoteS3Resource); + } + } + } + + private void checkAndRestoreStoragePolicies() { + PolicyMgr policyMgr = Env.getCurrentEnv().getPolicyMgr(); + for (BackupJobInfo.BackupStoragePolicyInfo backupStoragePolicyInfo : jobInfo.newBackupObjects.storagePolicies) { + String backupStoragePoliceName = backupStoragePolicyInfo.name; + Optional localPolicy = policyMgr.findPolicy(backupStoragePoliceName, + PolicyTypeEnum.STORAGE); + StoragePolicy backupStoargePolicy = backupMeta.getStoragePolicy(backupStoragePoliceName); + if (localPolicy.isPresent()) { + StoragePolicy localStoargePolicy = (StoragePolicy) localPolicy.get(); + if (localStoargePolicy.getVersion() + != backupStoargePolicy.getVersion()) { + status = new Status(ErrCode.COMMON_ERROR, "Storage policy " + + jobInfo.getAliasByOriginNameIfSet(backupStoragePoliceName) + + " already exist but with different properties"); + return; + } + + } else { + // restore storage policy + try { + policyMgr.replayCreate(backupStoargePolicy); + Env.getCurrentEnv().getEditLog().logCreatePolicy(backupStoargePolicy); + } catch (Exception e) { + LOG.error("restore user property fail should not happen", e); + } + storagePolicies.add(backupStoargePolicy); + } + } } private boolean genFileMappingWhenBackupReplicasEqual(PartitionInfo localPartInfo, Partition localPartition, @@ -1630,6 +1719,12 @@ private void replayCheckAndPrepareMeta() { resourceMgr.replayCreateResource(resource); } + // restored storage policy + PolicyMgr policyMgr = Env.getCurrentEnv().getPolicyMgr(); + for (StoragePolicy storagePolicy : storagePolicies) { + policyMgr.replayCreate(storagePolicy); + } + LOG.info("replay check and prepare meta. {}", this); } @@ -2100,6 +2195,7 @@ private Status allTabletCommitted(boolean isReplay) { restoredPartitions.clear(); restoredTbls.clear(); restoredResources.clear(); + storagePolicies.clear(); // release snapshot before clearing snapshotInfos releaseSnapshots(); @@ -2367,6 +2463,13 @@ private void cancelInternal(boolean isReplay) { LOG.info("remove restored resource when cancelled: {}", resource.getName()); resourceMgr.dropResource(resource); } + + // remove restored storage policy + PolicyMgr policyMgr = Env.getCurrentEnv().getPolicyMgr(); + for (StoragePolicy storagePolicy : storagePolicies) { + LOG.info("remove restored storage polciy when cancelled: {}", storagePolicy.getName()); + policyMgr.replayDrop(storagePolicy); + } } if (!isReplay) { @@ -2566,6 +2669,23 @@ public void readFields(DataInput in) throws IOException { } else { readOthers(in); } + + out.writeInt(restoredResources.size()); + for (Resource resource : restoredResources) { + resource.write(out); + } + + out.writeInt(storagePolicies.size()); + for (StoragePolicy policy : storagePolicies) { + policy.write(out); + } + + // write properties + out.writeInt(properties.size()); + for (Map.Entry entry : properties.entrySet()) { + Text.writeString(out, entry.getKey()); + Text.writeString(out, entry.getValue()); + } } private void readOthers(DataInput in) throws IOException { @@ -2634,6 +2754,12 @@ private void readOthers(DataInput in) throws IOException { restoredResources.add(Resource.read(in)); } + // restored storage policy + size = in.readInt(); + for (int i = 0; i < size; i++) { + storagePolicies.add(StoragePolicy.read(in)); + } + // read properties size = in.readInt(); for (int i = 0; i < size; i++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java index e6fac7f07d81ad..e0fee915eaa935 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java @@ -82,6 +82,13 @@ public void createResource(CreateResourceStmt stmt) throws DdlException { } } + public void createResource(Resource resource) throws DdlException { + if (createResource(resource, false)) { + Env.getCurrentEnv().getEditLog().logCreateResource(resource); + LOG.info("Create resource success. Resource: {}", resource.getName()); + } + } + // Return true if the resource is truly added, // otherwise, return false or throw exception. public boolean createResource(Resource resource, boolean ifNotExists) throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index a40e91f47d46d5..0e4c36ba19504f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -33,11 +33,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.UnsupportedEncodingException; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.zip.Adler32; /** * S3 resource @@ -239,5 +241,40 @@ protected void getProcNodeData(BaseProcResult result) { } readUnlock(); } + + public int getSignature(int signatureVersion) { + Adler32 adler32 = new Adler32(); + adler32.update(signatureVersion); + final String charsetName = "UTF-8"; + + try { + // table name + adler32.update(name.getBytes(charsetName)); + if (LOG.isDebugEnabled()) { + LOG.debug("signature. view name: {}", name); + } + // type + adler32.update(type.name().getBytes(charsetName)); + if (LOG.isDebugEnabled()) { + LOG.debug("signature. view type: {}", type.name()); + } + // configs + for (Map.Entry config : properties.entrySet()) { + adler32.update(config.getKey().getBytes(charsetName)); + adler32.update(config.getValue().getBytes(charsetName)); + if (LOG.isDebugEnabled()) { + LOG.debug("signature. view config: {}", config); + } + } + } catch (UnsupportedEncodingException e) { + LOG.error("encoding error", e); + return -1; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("signature: {}", Math.abs((int) adler32.getValue())); + } + return Math.abs((int) adler32.getValue()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java index 01c5399d4ab974..4867a5c62f9676 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java @@ -99,6 +99,10 @@ public Policy(long id, final PolicyTypeEnum type, final String policyName) { this.version = 0; } + public String getName() { + return policyName; + } + /** * Trans stmt to Policy. **/ diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java index 6e8bd4f08cb2f7..f1c8dd27f8d3b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java @@ -272,7 +272,7 @@ public List getCopiedPoliciesByType(PolicyTypeEnum policyType) { } } - private List getPoliciesByType(PolicyTypeEnum policyType) { + public List getPoliciesByType(PolicyTypeEnum policyType) { if (typeToPolicyMap == null) { return new ArrayList<>(); } @@ -319,6 +319,11 @@ private void unprotectedAdd(Policy policy) { } + public void replayDrop(StoragePolicy policy) { + DropPolicyLog log = new DropPolicyLog(policy.getType(), policy.getPolicyName()); + replayDrop(log); + } + public void replayDrop(DropPolicyLog log) { // for compatible if (log.getType() == PolicyTypeEnum.ROW && StringUtils.isEmpty(log.getCtlName())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java index ba2e0d5c59218e..ea4ed4f7580142 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java @@ -25,8 +25,12 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.persist.gson.GsonPostProcessable; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -37,6 +41,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.time.LocalDateTime; import java.time.format.DateTimeParseException; @@ -48,7 +54,7 @@ * Save policy for storage migration. **/ @Data -public class StoragePolicy extends Policy { +public class StoragePolicy extends Policy implements Writable, GsonPostProcessable { public static final String DEFAULT_STORAGE_POLICY_NAME = "default_storage_policy"; public static boolean checkDefaultStoragePolicyValid(final String storagePolicyName, Optional defaultPolicy) @@ -185,6 +191,20 @@ public void init(final Map props, boolean ifNotExists) throws An } } + @Override + public void write(DataOutput out) throws IOException { + String json = GsonUtils.GSON.toJson(this); + Text.writeString(out, json); + } + + /** + * Read Policy from file. + **/ + public static StoragePolicy read(DataInput in) throws IOException { + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, StoragePolicy.class); + } + private static Resource checkResourceIsExist(final String storageResource) throws AnalysisException { Resource resource = Optional.ofNullable(Env.getCurrentEnv().getResourceMgr().getResource(storageResource)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 1ad8d733ddea07..3d7ed79524b738 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -28,6 +28,8 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.backup.AbstractJob; +import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Snapshot; import org.apache.doris.catalog.AutoIncrementGenerator; import org.apache.doris.catalog.Column; @@ -343,6 +345,19 @@ public TConfirmUnusedRemoteFilesResult confirmUnusedRemoteFiles(TConfirmUnusedRe LOG.warn("tablet {} not found", info.tablet_id); return; } + + List jobs = Env.getCurrentEnv().getBackupHandler() + .getJobs(tabletMeta.getDbId(), label -> true); + + List runningBackupJobs = jobs.stream().filter(job -> job instanceof BackupJob) + .filter(job -> !((BackupJob) job).isDone()) + .map(job -> (BackupJob) job).collect(Collectors.toList()); + + if (runningBackupJobs.size() > 0) { + LOG.warn("Backup is running on this tablet {} ", info.tablet_id); + return; + } + Tablet tablet; int replicaNum; try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java index ba564599029994..cb929668bf5a71 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java @@ -43,6 +43,7 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.persist.EditLog; +import org.apache.doris.policy.StoragePolicy; import org.apache.doris.task.DirMoveTask; import org.apache.doris.task.DownloadTask; import org.apache.doris.task.SnapshotTask; @@ -213,7 +214,8 @@ public Status getSnapshotInfoFile(String label, String backupTimestamp, List tbls = Lists.newArrayList(); tbls.add(tbl); List resources = Lists.newArrayList(); - BackupMeta backupMeta = new BackupMeta(tbls, resources); + List storagePolicys = Lists.newArrayList(); + BackupMeta backupMeta = new BackupMeta(tbls, resources, storagePolicys); Map snapshotInfos = Maps.newHashMap(); for (Partition part : tbl.getPartitions()) { for (MaterializedIndex idx : part.getMaterializedIndices(IndexExtState.VISIBLE)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java index dadfdb632e394d..c5473b7739b255 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java @@ -42,6 +42,7 @@ import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.persist.EditLog; +import org.apache.doris.policy.StoragePolicy; import org.apache.doris.resource.Tag; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TStorageMedium; @@ -261,8 +262,9 @@ boolean await(long timeout, TimeUnit unit) { List
tbls = Lists.newArrayList(); List resources = Lists.newArrayList(); + List storagePolicys = Lists.newArrayList(); tbls.add(expectedRestoreTbl); - backupMeta = new BackupMeta(tbls, resources); + backupMeta = new BackupMeta(tbls, resources, storagePolicys); } @Test diff --git a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy new file mode 100644 index 00000000000000..ea503a8cbaf539 --- /dev/null +++ b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_cooldown", "backup_cooldown_data") { + + String suiteName = "test_backup_cooldown" + String resource_name1 = "resource_${suiteName}_1" + String policy_name1 = "policy_${suiteName}_1" + String resource_name2 = "resource_${suiteName}_2" + String policy_name2 = "policy_${suiteName}_2" + String dbName = "${suiteName}_db" + String tableName = "${suiteName}_table" + String snapshotName = "${suiteName}_snapshot" + String repoName = "${suiteName}_repo" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + + + + sql """ + CREATE RESOURCE IF NOT EXISTS "${resource_name1}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "AWS_REGION" = "${getS3Region()}", + "AWS_ROOT_PATH" = "regression/cooldown1", + "AWS_ACCESS_KEY" = "${getS3AK()}", + "AWS_SECRET_KEY" = "${getS3SK()}", + "AWS_MAX_CONNECTIONS" = "50", + "AWS_REQUEST_TIMEOUT_MS" = "3000", + "AWS_CONNECTION_TIMEOUT_MS" = "1000", + "AWS_BUCKET" = "${getS3BucketName()}", + "s3_validity_check" = "true" + ); + """ + + sql """ + CREATE RESOURCE IF NOT EXISTS "${resource_name2}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "AWS_REGION" = "${getS3Region()}", + "AWS_ROOT_PATH" = "regression/cooldown2", + "AWS_ACCESS_KEY" = "${getS3AK()}", + "AWS_SECRET_KEY" = "${getS3SK()}", + "AWS_MAX_CONNECTIONS" = "50", + "AWS_REQUEST_TIMEOUT_MS" = "3000", + "AWS_CONNECTION_TIMEOUT_MS" = "1000", + "AWS_BUCKET" = "${getS3BucketName()}", + "s3_validity_check" = "true" + ); + """ + + sql """ + CREATE STORAGE POLICY IF NOT EXISTS ${policy_name1} + PROPERTIES( + "storage_resource" = "${resource_name1}", + "cooldown_ttl" = "10" + ) + """ + + sql """ + CREATE STORAGE POLICY IF NOT EXISTS ${policy_name2} + PROPERTIES( + "storage_resource" = "${resource_name2}", + "cooldown_ttl" = "10" + ) + """ + + //generate_cooldown_task_interval_sec default is 20 + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + + sql """ + CREATE TABLE ${dbName}.${tableName} + ( + k1 BIGINT, + v1 VARCHAR(48), + INDEX idx1 (v1) USING INVERTED PROPERTIES("parser" = "english") + ) + DUPLICATE KEY(k1) + PARTITION BY RANGE(`k1`) + ( + PARTITION p201701 VALUES [(0), (3)) ("storage_policy" = "${policy_name1}"), + PARTITION `p201702` VALUES LESS THAN (6)("storage_policy" = "${policy_name2}"), + PARTITION `p2018` VALUES [(6),(100)) + ) + DISTRIBUTED BY HASH (k1) BUCKETS 3 + PROPERTIES( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + List values = [] + for (int i = 1; i <= 10; ++i) { + values.add("(${i}, ${i})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + int count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) + + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + + + } + + assertNotEquals('0.000 ', result[0][5].toString()) + + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + syncer.waitSnapshotFinish(dbName) + + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "DROP TABLE ${dbName}.${tableName}" + + sql """ + drop storage policy ${policy_name1}; + """ + + sql """ + drop resource ${resource_name1}; + """ + + sql """ + drop storage policy ${policy_name2}; + """ + + sql """ + drop resource ${resource_name2}; + """ + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) + + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + + + } + + //cleanup + sql "DROP TABLE ${dbName}.${tableName} FORCE" + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP REPOSITORY `${repoName}`" + + sql """ + drop storage policy ${policy_name1}; + """ + + sql """ + drop resource ${resource_name1}; + """ + + sql """ + drop storage policy ${policy_name2}; + """ + + sql """ + drop resource ${resource_name2}; + """ +} \ No newline at end of file From de678d2933b123e8bf8dc81cd2a3782887274f4b Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Thu, 12 Dec 2024 18:57:10 +0800 Subject: [PATCH 02/15] fix for confict with master --- be/src/io/fs/s3_file_system.h | 8 +------ be/src/olap/rowset/beta_rowset.cpp | 37 +++++++++++++++++------------- be/src/olap/rowset/beta_rowset.h | 4 ++-- be/src/olap/rowset/rowset.h | 8 ++----- be/src/olap/tablet.cpp | 5 ++-- be/src/runtime/snapshot_loader.cpp | 15 ++++++------ 6 files changed, 35 insertions(+), 42 deletions(-) diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h index ed671425468f1a..31d2e19a1a9a38 100644 --- a/be/src/io/fs/s3_file_system.h +++ b/be/src/io/fs/s3_file_system.h @@ -120,13 +120,7 @@ class S3FileSystem final : public RemoteFileSystem { // so no need to concat with prefix abs_path = path; } else { - std::string path_str = _root_path.string(); - if (!path_str.empty() && path_str[0] == '/') { - path_str.erase(0, 1); // remove first '/' - } - // path with no schema - abs_path = _prefix / path; - return std::filesystem::path(path_str) / path; + return std::filesystem::path(fmt::format("s3://{}/{}", _bucket, _prefix)) / path; } return Status::OK(); } diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 4196a79305ae06..0243ec832d0a18 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -432,7 +432,7 @@ Status BetaRowset::copy_files_to(const std::string& dir, const RowsetId& new_row return Status::OK(); } -Status BetaRowset::download(io::RemoteFileSystem* fs, const std::string& dir) { +Status BetaRowset::download(const StorageResource& dest_fs, const std::string& dir) { if (is_local()) { DCHECK(false) << _rowset_meta->tablet_id() << ' ' << rowset_id(); return Status::InternalError("should be remote rowset. tablet_id={} rowset_id={}", @@ -453,7 +453,7 @@ Status BetaRowset::download(io::RemoteFileSystem* fs, const std::string& dir) { paths.emplace_back(file); LOG(WARNING) << "will delete download success file " << file << " due to error"; } - static_cast(fs->batch_delete(paths)); + static_cast(dest_fs.fs->batch_delete(paths)); LOG(WARNING) << "done delete download success files due to error " << status; } }}; @@ -461,43 +461,48 @@ Status BetaRowset::download(io::RemoteFileSystem* fs, const std::string& dir) { for (int i = 0; i < num_segments(); ++i) { // Note: Here we use relative path for remote. auto remote_seg_path = - remote_segment_path(_rowset_meta->tablet_id(), rowset_id().to_string(), i); + dest_fs.remote_segment_path(_rowset_meta->tablet_id(), rowset_id().to_string(), i); - auto local_seg_path = segment_file_path(dir, rowset_id(), i); + auto local_seg_path = local_segment_path(dir, rowset_id().to_string(), i); - RETURN_IF_ERROR(fs->download(remote_seg_path, local_seg_path)); + RETURN_IF_ERROR(dest_fs.fs->download(remote_seg_path, local_seg_path)); linked_success_files.push_back(local_seg_path); if (_schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1) { if (_schema->has_inverted_index()) { std::string inverted_index_src_file = - InvertedIndexDescriptor::get_index_file_name(remote_seg_path); + InvertedIndexDescriptor::get_index_file_path_v2( + InvertedIndexDescriptor::get_index_file_path_prefix(remote_seg_path)); std::string inverted_index_dst_file_path = - InvertedIndexDescriptor::get_index_file_name(local_seg_path); + InvertedIndexDescriptor::get_index_file_path_v2( + InvertedIndexDescriptor::get_index_file_path_prefix(local_seg_path)); RETURN_IF_ERROR( - fs->download(inverted_index_src_file, inverted_index_dst_file_path)); + dest_fs.fs->download(inverted_index_src_file, inverted_index_dst_file_path)); linked_success_files.push_back(inverted_index_dst_file_path); } } else { - for (const auto& index : _schema->indexes()) { - if (index.index_type() != IndexType::INVERTED) { + for (const auto& index : _schema->inverted_indexes()) { + if (index->index_type() != IndexType::INVERTED) { continue; } - auto index_id = index.index_id(); - std::string inverted_index_src_file = InvertedIndexDescriptor::get_index_file_name( - remote_seg_path, index_id, index.get_index_suffix()); + auto index_id = index->index_id(); + std::string inverted_index_src_file = + InvertedIndexDescriptor::get_index_file_path_v1( + InvertedIndexDescriptor::get_index_file_path_prefix(remote_seg_path), + index_id, index->get_index_suffix()); std::string inverted_index_dst_file_path = - InvertedIndexDescriptor::get_index_file_name(local_seg_path, index_id, - index.get_index_suffix()); + InvertedIndexDescriptor::get_index_file_path_v1( + InvertedIndexDescriptor::get_index_file_path_prefix(local_seg_path), + index_id, index->get_index_suffix()); RETURN_IF_ERROR( - fs->download(inverted_index_src_file, inverted_index_dst_file_path)); + dest_fs.fs->download(inverted_index_src_file, inverted_index_dst_file_path)); linked_success_files.push_back(inverted_index_dst_file_path); LOG(INFO) << "success to download. from=" << inverted_index_src_file << ", " diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index dd8d536a29bd87..100e7aa45e2dc7 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -63,9 +63,9 @@ class BetaRowset final : public Rowset { Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) override; - Status download(io::RemoteFileSystem* fs, const std::string& dir) override; + Status download(const StorageResource& dest_fs, const std::string& dir) override; - Status upload_to(io::RemoteFileSystem* dest_fs, const RowsetId& new_rowset_id) override; + Status upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) override; // only applicable to alpha rowset, no op here Status remove_old_files(std::vector* files_to_remove) override { diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index cdb370c809a1c9..4545c7ac6a1508 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -213,13 +213,9 @@ class Rowset : public std::enable_shared_from_this, public MetadataAdder // copy all files to `dir` virtual Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) = 0; - virtual Status download(io::RemoteFileSystem* fs, const std::string& dir) { - return Status::OK(); - } + virtual Status download(const StorageResource& dest_fs, const std::string& dir) = 0; - virtual Status upload_to(io::RemoteFileSystem* dest_fs, const RowsetId& new_rowset_id) { - return Status::OK(); - } + virtual Status upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) = 0; virtual Status remove_old_files(std::vector* files_to_remove) = 0; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index d75d1affadfebf..9f911be33f1ee9 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2026,11 +2026,10 @@ Status Tablet::cooldown(RowsetSharedPtr rowset) { } Status Tablet::download(RowsetSharedPtr rowset, const std::string& dir) { - std::shared_ptr dest_fs; - RETURN_IF_ERROR(get_remote_file_system(storage_policy_id(), &dest_fs)); Status st; + auto storage_resource = DORIS_TRY(get_resource_by_storage_policy_id(storage_policy_id())); - if (st = rowset->download(dest_fs.get(), dir); !st.ok()) { + if (st = rowset->download(storage_resource, dir); !st.ok()) { return st; } diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index 25c1aef1ad2c1b..e0c89bba8656f7 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -167,9 +167,11 @@ static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet int segments, int have_inverted_index) { Status res = Status::OK(); + std::string remote_tablet_path = fmt::format("{}/{}", DATA_PREFIX, tablet_id); + for (int i = 0; i < segments; i++) { std::string remote_seg_path = - fmt::format("{}/{}_{}.dat", remote_tablet_path(tablet_id), rowset, i); + fmt::format("{}/{}_{}.dat", remote_tablet_path, rowset, i); std::string local_seg_path = fmt::format("{}/{}_{}.dat", local_path, rowset, i); std::string dest_seg_path = fmt::format("{}/{}_{}.dat", dest_path, rowset, i); @@ -182,12 +184,12 @@ static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet } std::vector remote_index_files; - RETURN_IF_ERROR(list_segment_inverted_index_file(cold_fs, remote_tablet_path(tablet_id), rowset, + RETURN_IF_ERROR(list_segment_inverted_index_file(cold_fs, remote_tablet_path, rowset, &remote_index_files)); for (auto& index_file : remote_index_files) { std::string remote_index_path = - fmt::format("{}/{}", remote_tablet_path(tablet_id), index_file); + fmt::format("{}/{}", remote_tablet_path, index_file); std::string local_seg_path = fmt::format("{}/{}", local_path, index_file); std::string dest_seg_path = fmt::format("{}/{}", dest_path, index_file); @@ -227,8 +229,7 @@ static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_i int segments; int have_inverted_index; - std::shared_ptr colddata_fs; - RETURN_IF_ERROR(get_remote_file_system(storage_policy_id, &colddata_fs)); + auto storage_resource = DORIS_TRY(get_resource_by_storage_policy_id(storage_policy_id)); while (end != std::string::npos) { end = str.find(delimiter, start); // @@ -245,7 +246,7 @@ static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_i if (segments > 0) { RETURN_IF_ERROR(upload_remote_rowset(dest_fs, tablet_id, local_path, dest_path, - colddata_fs.get(), rowset_id, segments, + storage_resource.fs.get(), rowset_id, segments, have_inverted_index)); } } @@ -301,8 +302,6 @@ Status SnapshotLoader::upload(const std::map& src_to_d for (auto& local_file : local_files) { RETURN_IF_ERROR(_report_every(10, &report_counter, finished_num, total_num, TTaskType::type::UPLOAD)); - - const std::string& local_file = *it; if (local_file.compare("remote_file_info") == 0) { RETURN_IF_ERROR(upload_remote_file(*_remote_fs, tablet_id, src_path, dest_path, local_file)); From 6cb98626f22130ce1bdba07a6861dd07036f4eee Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Thu, 7 Nov 2024 17:27:39 +0800 Subject: [PATCH 03/15] fix check storage policy --- .../org/apache/doris/backup/RestoreJob.java | 26 ++++++++- .../org/apache/doris/catalog/OlapTable.java | 2 +- .../apache/doris/policy/StoragePolicy.java | 54 +++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index a3606cfd09ab74..eca592968761fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -688,6 +688,18 @@ private void checkAndPrepareMeta() { } } + for (BackupJobInfo.BackupStoragePolicyInfo backupStoragePolicyInfo : jobInfo.newBackupObjects.storagePolicies) { + String backupStoragePoliceName = backupStoragePolicyInfo.name; + Optional localPolicy = Env.getCurrentEnv().getPolicyMgr().findPolicy(backupStoragePoliceName, + PolicyTypeEnum.STORAGE); + if (localPolicy.isPresent() && localPolicy.get().getType() != PolicyTypeEnum.STORAGE) { + status = new Status(ErrCode.COMMON_ERROR, + "The local policy " + backupStoragePoliceName + + " with the same name but a different type of backup meta."); + return; + } + } + // the new tablets -> { local tablet, schema hash, storage medium }, used in atomic restore. Map tabletBases = new HashMap<>(); @@ -1338,6 +1350,14 @@ private void checkAndRestoreStoragePolicies() { + " already exist but with different properties"); return; } + // storage policy id should be same + if (localStoargePolicy.getSignature(BackupHandler.SIGNATURE_VERSION) + != backupStoargePolicy.getSignature(BackupHandler.SIGNATURE_VERSION)) { + status = new Status(ErrCode.COMMON_ERROR, "Storage policy " + + jobInfo.getAliasByOriginNameIfSet(backupStoragePoliceName) + + " already exist but with different properties"); + return; + } } else { // restore storage policy @@ -1722,7 +1742,11 @@ private void replayCheckAndPrepareMeta() { // restored storage policy PolicyMgr policyMgr = Env.getCurrentEnv().getPolicyMgr(); for (StoragePolicy storagePolicy : storagePolicies) { - policyMgr.replayCreate(storagePolicy); + Optional localPolicy = policyMgr.findPolicy(storagePolicy.getPolicyName(), + PolicyTypeEnum.STORAGE); + if (!localPolicy.isPresent()) { + policyMgr.replayCreate(storagePolicy); + } } LOG.info("replay check and prepare meta. {}", this); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 92d9aa4e9c7f82..840311028ead72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2011,7 +2011,7 @@ public OlapTable selectiveCopy(Collection reservedPartitions, IndexExtSt // set storage medium to HDD for backup job, because we want that the backuped table // can be able to restored to another Doris cluster without SSD disk. // But for other operation such as truncate table, keep the origin storage medium. - copied.getPartitionInfo().setDataProperty(partition.getId(), new DataProperty(TStorageMedium.HDD)); + copied.getPartitionInfo().getDataProperty(partition.getId()).setStorageMedium(TStorageMedium.HDD); } for (MaterializedIndex idx : partition.getMaterializedIndices(extState)) { idx.setState(IndexState.NORMAL); diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java index ea4ed4f7580142..6d3c8e7f797d3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java @@ -44,11 +44,13 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.time.LocalDateTime; import java.time.format.DateTimeParseException; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.zip.Adler32; /** * Save policy for storage migration. @@ -411,4 +413,56 @@ public boolean removeResourceReference() { } return false; } + + public int getSignature(int signatureVersion) { + Adler32 adler32 = new Adler32(); + adler32.update(signatureVersion); + final String charsetName = "UTF-8"; + + try { + // policy name + adler32.update(policyName.getBytes(charsetName)); + if (LOG.isDebugEnabled()) { + LOG.debug("signature. policy name: {}", policyName); + } + // storageResource name + adler32.update(policyName.getBytes(charsetName)); + if (LOG.isDebugEnabled()) { + LOG.debug("signature. storageResource name: {}", storageResource); + } + // id + adler32.update(String.valueOf(id).getBytes(charsetName)); + if (LOG.isDebugEnabled()) { + LOG.debug("signature. id : {}", id); + } + // type + adler32.update(String.valueOf(getType()).getBytes(charsetName)); + if (LOG.isDebugEnabled()) { + LOG.debug("signature. type : {}", getType()); + } + // version + adler32.update(String.valueOf(getVersion()).getBytes(charsetName)); + if (LOG.isDebugEnabled()) { + LOG.debug("signature. version : {}", getVersion()); + } + // cooldownTimestampMs + adler32.update(String.valueOf(cooldownTimestampMs).getBytes(charsetName)); + if (LOG.isDebugEnabled()) { + LOG.debug("signature. cooldownTimestampMs : {}", cooldownTimestampMs); + } + // cooldownTtl + adler32.update(String.valueOf(cooldownTtl).getBytes(charsetName)); + if (LOG.isDebugEnabled()) { + LOG.debug("signature. cooldownTtl : {}", cooldownTtl); + } + } catch (UnsupportedEncodingException e) { + LOG.error("encoding error", e); + return -1; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("signature: {}", Math.abs((int) adler32.getValue())); + } + return Math.abs((int) adler32.getValue()); + } } From 929a7bbc84d9470ef6d63ea2e4a1ef7c3bfb6668 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Mon, 11 Nov 2024 11:29:53 +0800 Subject: [PATCH 04/15] fix s3 absolute_path --- be/src/olap/rowset/beta_rowset.cpp | 23 +- be/src/olap/rowset/beta_rowset.h | 1 - be/src/olap/single_replica_compaction.cpp | 2 +- be/src/olap/snapshot_manager.cpp | 3 +- be/src/olap/snapshot_manager.h | 8 +- be/src/olap/task/engine_clone_task.cpp | 2 +- .../task/engine_storage_migration_task.cpp | 2 +- be/src/runtime/snapshot_loader.cpp | 12 +- .../apache/doris/analysis/RestoreStmt.java | 35 + .../apache/doris/backup/BackupHandler.java | 6 +- .../org/apache/doris/backup/RestoreJob.java | 175 ++- .../org/apache/doris/catalog/OlapTable.java | 4 +- .../apache/doris/catalog/PartitionInfo.java | 8 +- .../apache/doris/policy/StoragePolicy.java | 23 +- .../apache/doris/backup/RestoreJobTest.java | 2 +- .../test_backup_restore_cold_data.groovy | 1152 +++++++++++++++++ 16 files changed, 1339 insertions(+), 119 deletions(-) diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 0243ec832d0a18..7d0ddd94c42233 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -473,14 +473,16 @@ Status BetaRowset::download(const StorageResource& dest_fs, const std::string& d if (_schema->has_inverted_index()) { std::string inverted_index_src_file = InvertedIndexDescriptor::get_index_file_path_v2( - InvertedIndexDescriptor::get_index_file_path_prefix(remote_seg_path)); + InvertedIndexDescriptor::get_index_file_path_prefix( + remote_seg_path)); std::string inverted_index_dst_file_path = InvertedIndexDescriptor::get_index_file_path_v2( - InvertedIndexDescriptor::get_index_file_path_prefix(local_seg_path)); + InvertedIndexDescriptor::get_index_file_path_prefix( + local_seg_path)); - RETURN_IF_ERROR( - dest_fs.fs->download(inverted_index_src_file, inverted_index_dst_file_path)); + RETURN_IF_ERROR(dest_fs.fs->download(inverted_index_src_file, + inverted_index_dst_file_path)); linked_success_files.push_back(inverted_index_dst_file_path); } @@ -493,16 +495,17 @@ Status BetaRowset::download(const StorageResource& dest_fs, const std::string& d auto index_id = index->index_id(); std::string inverted_index_src_file = InvertedIndexDescriptor::get_index_file_path_v1( - InvertedIndexDescriptor::get_index_file_path_prefix(remote_seg_path), - index_id, index->get_index_suffix()); + InvertedIndexDescriptor::get_index_file_path_prefix( + remote_seg_path), + index_id, index->get_index_suffix()); std::string inverted_index_dst_file_path = InvertedIndexDescriptor::get_index_file_path_v1( - InvertedIndexDescriptor::get_index_file_path_prefix(local_seg_path), - index_id, index->get_index_suffix()); + InvertedIndexDescriptor::get_index_file_path_prefix(local_seg_path), + index_id, index->get_index_suffix()); - RETURN_IF_ERROR( - dest_fs.fs->download(inverted_index_src_file, inverted_index_dst_file_path)); + RETURN_IF_ERROR(dest_fs.fs->download(inverted_index_src_file, + inverted_index_dst_file_path)); linked_success_files.push_back(inverted_index_dst_file_path); LOG(INFO) << "success to download. from=" << inverted_index_src_file << ", " diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 100e7aa45e2dc7..4d36bb251de79e 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -62,7 +62,6 @@ class BetaRowset final : public Rowset { Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) override; - Status download(const StorageResource& dest_fs, const std::string& dir) override; Status upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) override; diff --git a/be/src/olap/single_replica_compaction.cpp b/be/src/olap/single_replica_compaction.cpp index 458f3949b17017..0f8fc141790fd3 100644 --- a/be/src/olap/single_replica_compaction.cpp +++ b/be/src/olap/single_replica_compaction.cpp @@ -321,7 +321,7 @@ Status SingleReplicaCompaction::_fetch_rowset(const TReplicaInfo& addr, const st RETURN_IF_ERROR(_download_files(tablet()->data_dir(), remote_url_prefix, local_path)); _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( local_path, _tablet->tablet_id(), tablet()->replica_id(), _tablet->table_id(), - _tablet->partition_id(), _tablet->schema_hash())); + _tablet->partition_id(), _tablet->schema_hash(), 0)); // 4: finish_clone: create output_rowset and link file return _finish_clone(local_data_path, rowset_version); } diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index c8adff009a2d9f..4720415a553819 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -141,7 +141,7 @@ Status SnapshotManager::release_snapshot(const string& snapshot_path) { Result> SnapshotManager::convert_rowset_ids( const std::string& clone_dir, int64_t tablet_id, int64_t replica_id, int64_t table_id, - int64_t partition_id, int32_t schema_hash) { + int64_t partition_id, int32_t schema_hash, int64_t storage_policy_id) { SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker); std::vector guards; // check clone dir existed @@ -171,6 +171,7 @@ Result> SnapshotManager::convert_rowset_ids( new_tablet_meta_pb.set_tablet_id(tablet_id); *new_tablet_meta_pb.mutable_tablet_uid() = TabletUid::gen_uid().to_proto(); new_tablet_meta_pb.set_replica_id(replica_id); + new_tablet_meta_pb.set_storage_policy_id(storage_policy_id); if (table_id > 0) { new_tablet_meta_pb.set_table_id(table_id); } diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h index dd10f7f355058b..160b67f6befd1e 100644 --- a/be/src/olap/snapshot_manager.h +++ b/be/src/olap/snapshot_manager.h @@ -51,11 +51,9 @@ class SnapshotManager { // @param snapshot_path [in] 要被释放的snapshot的路径，只包含到ID Status release_snapshot(const std::string& snapshot_path); - Result> convert_rowset_ids(const std::string& clone_dir, - int64_t tablet_id, - int64_t replica_id, int64_t table_id, - int64_t partition_id, - int32_t schema_hash); + Result> convert_rowset_ids( + const std::string& clone_dir, int64_t tablet_id, int64_t replica_id, int64_t table_id, + int64_t partition_id, int32_t schema_hash, int64_t storage_policy_id); private: Status _calc_snapshot_id_path(const TabletSharedPtr& tablet, int64_t timeout_s, diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index fa8d9b8248e3f4..64e9ca283ccc3e 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -459,7 +459,7 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir, // No need to try again with another BE _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( local_data_path, _clone_req.tablet_id, _clone_req.replica_id, _clone_req.table_id, - _clone_req.partition_id, _clone_req.schema_hash)); + _clone_req.partition_id, _clone_req.schema_hash, 0)); break; } // clone copy from one backend return status; diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index a300e6e0f09fa3..cf9ac35b3f67a3 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -161,7 +161,7 @@ Status EngineStorageMigrationTask::_gen_and_write_header_to_hdr_file( // rowset create time is useful when load tablet from meta to check which tablet is the tablet to load _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( full_path, tablet_id, _tablet->replica_id(), _tablet->table_id(), - _tablet->partition_id(), schema_hash)); + _tablet->partition_id(), schema_hash, 0)); return Status::OK(); } diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index e0c89bba8656f7..bfe8618d1e976c 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -170,8 +170,7 @@ static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet std::string remote_tablet_path = fmt::format("{}/{}", DATA_PREFIX, tablet_id); for (int i = 0; i < segments; i++) { - std::string remote_seg_path = - fmt::format("{}/{}_{}.dat", remote_tablet_path, rowset, i); + std::string remote_seg_path = fmt::format("{}/{}_{}.dat", remote_tablet_path, rowset, i); std::string local_seg_path = fmt::format("{}/{}_{}.dat", local_path, rowset, i); std::string dest_seg_path = fmt::format("{}/{}_{}.dat", dest_path, rowset, i); @@ -188,8 +187,7 @@ static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet &remote_index_files)); for (auto& index_file : remote_index_files) { - std::string remote_index_path = - fmt::format("{}/{}", remote_tablet_path, index_file); + std::string remote_index_path = fmt::format("{}/{}", remote_tablet_path, index_file); std::string local_seg_path = fmt::format("{}/{}", local_path, index_file); std::string dest_seg_path = fmt::format("{}/{}", dest_path, index_file); @@ -896,9 +894,9 @@ Status SnapshotLoader::move(const std::string& snapshot_path, TabletSharedPtr ta } // rename the rowset ids and tabletid info in rowset meta - auto res = _engine.snapshot_mgr()->convert_rowset_ids(snapshot_path, tablet_id, - tablet->replica_id(), tablet->table_id(), - tablet->partition_id(), schema_hash); + auto res = _engine.snapshot_mgr()->convert_rowset_ids( + snapshot_path, tablet_id, tablet->replica_id(), tablet->table_id(), + tablet->partition_id(), schema_hash, tablet->storage_policy_id()); if (!res.has_value()) [[unlikely]] { auto err_msg = fmt::format("failed to convert rowsetids in snapshot: {}, tablet path: {}, err: {}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java index bc38cfe09e5606..2287ea55be9b50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java @@ -18,7 +18,9 @@ package org.apache.doris.analysis; import org.apache.doris.backup.Repository; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ReplicaAllocation; +import org.apache.doris.catalog.Resource; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; @@ -44,18 +46,22 @@ public class RestoreStmt extends AbstractBackupStmt implements NotFallbackInPars public static final String PROP_CLEAN_TABLES = "clean_tables"; public static final String PROP_CLEAN_PARTITIONS = "clean_partitions"; public static final String PROP_ATOMIC_RESTORE = "atomic_restore"; + public static final String PROP_STORAGE_SOURCE = "storage_resource"; + public static final String PROP_RESERVE_STORAGE_POLICY = "reserve_storage_policy"; private boolean allowLoad = false; private ReplicaAllocation replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION; private String backupTimestamp = null; private int metaVersion = -1; private boolean reserveReplica = false; + private boolean reserveStoragePolicy = true; private boolean reserveDynamicPartitionEnable = false; private boolean isLocal = false; private boolean isBeingSynced = false; private boolean isCleanTables = false; private boolean isCleanPartitions = false; private boolean isAtomicRestore = false; + private String storageResource = null; private byte[] meta = null; private byte[] jobInfo = null; @@ -83,6 +89,10 @@ public String getBackupTimestamp() { return backupTimestamp; } + public String getStorageResource() { + return storageResource; + } + public int getMetaVersion() { return metaVersion; } @@ -91,6 +101,10 @@ public boolean reserveReplica() { return reserveReplica; } + public boolean reserveStoragePolicy() { + return reserveStoragePolicy; + } + public boolean reserveDynamicPartitionEnable() { return reserveDynamicPartitionEnable; } @@ -212,6 +226,27 @@ public void analyzeProperties() throws AnalysisException { // is atomic restore isAtomicRestore = eatBooleanProperty(copiedProperties, PROP_ATOMIC_RESTORE, isAtomicRestore); + if (copiedProperties.containsKey(PROP_STORAGE_SOURCE)) { + storageResource = copiedProperties.get(PROP_STORAGE_SOURCE); + Resource localResource = Env.getCurrentEnv().getResourceMgr().getResource(storageResource); + + if (localResource == null) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, + "Restore storage resource " + storageResource + " is not exist"); + } + + if (localResource.getType() != Resource.ResourceType.S3) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, + "The type of local resource " + + storageResource + " is not same as restored resource"); + } + + copiedProperties.remove(PROP_STORAGE_SOURCE); + } + + // reserve storage policy + reserveStoragePolicy = eatBooleanProperty(copiedProperties, PROP_RESERVE_STORAGE_POLICY, reserveStoragePolicy); + if (!copiedProperties.isEmpty()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, "Unknown restore job properties: " + copiedProperties.keySet()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 6f88881e3cb2a3..57c38cc6c50bba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -560,14 +560,14 @@ private void restore(Repository repository, Database db, RestoreStmt stmt) throw db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(), stmt.getTimeoutMs(), metaVersion, stmt.reserveReplica(), stmt.reserveDynamicPartitionEnable(), stmt.isBeingSynced(), - stmt.isCleanTables(), stmt.isCleanPartitions(), stmt.isAtomicRestore(), - env, Repository.KEEP_ON_LOCAL_REPO_ID, backupMeta); + stmt.isCleanTables(), stmt.isCleanPartitions(), stmt.isAtomicRestore(), stmt.getStorageResource(), + stmt.reserveStoragePolicy(), env, Repository.KEEP_ON_LOCAL_REPO_ID, backupMeta); } else { restoreJob = new RestoreJob(stmt.getLabel(), stmt.getBackupTimestamp(), db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(), stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(), stmt.reserveDynamicPartitionEnable(), stmt.isBeingSynced(), stmt.isCleanTables(), stmt.isCleanPartitions(), stmt.isAtomicRestore(), - env, repository.getId()); + stmt.getStorageResource(), stmt.reserveStoragePolicy(), env, repository.getId()); } env.getEditLog().logRestoreJob(restoreJob); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index eca592968761fa..6f5b80b7c56370 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -238,7 +238,8 @@ public RestoreJob(JobType jobType) { public RestoreJob(String label, String backupTs, long dbId, String dbName, BackupJobInfo jobInfo, boolean allowLoad, ReplicaAllocation replicaAlloc, long timeoutMs, int metaVersion, boolean reserveReplica, boolean reserveDynamicPartitionEnable, boolean isBeingSynced, boolean isCleanTables, - boolean isCleanPartitions, boolean isAtomicRestore, Env env, long repoId) { + boolean isCleanPartitions, boolean isAtomicRestore, String storageResource, + boolean reserveStoragePolicy, Env env, long repoId) { super(JobType.RESTORE, label, dbId, dbName, timeoutMs, env, repoId); this.backupTimestamp = backupTs; this.jobInfo = jobInfo; @@ -256,6 +257,8 @@ public RestoreJob(String label, String backupTs, long dbId, String dbName, Backu this.isCleanTables = isCleanTables; this.isCleanPartitions = isCleanPartitions; this.isAtomicRestore = isAtomicRestore; + this.storageResource = storageResource; + this.reserveStoragePolicy = reserveStoragePolicy; properties.put(PROP_RESERVE_REPLICA, String.valueOf(reserveReplica)); properties.put(PROP_RESERVE_DYNAMIC_PARTITION_ENABLE, String.valueOf(reserveDynamicPartitionEnable)); properties.put(PROP_IS_BEING_SYNCED, String.valueOf(isBeingSynced)); @@ -267,10 +270,11 @@ public RestoreJob(String label, String backupTs, long dbId, String dbName, Backu public RestoreJob(String label, String backupTs, long dbId, String dbName, BackupJobInfo jobInfo, boolean allowLoad, ReplicaAllocation replicaAlloc, long timeoutMs, int metaVersion, boolean reserveReplica, boolean reserveDynamicPartitionEnable, boolean isBeingSynced, boolean isCleanTables, - boolean isCleanPartitions, boolean isAtomicRestore, Env env, long repoId, BackupMeta backupMeta) { + boolean isCleanPartitions, boolean isAtomicRestore, String storageResource, + boolean reserveStoragePolicy, Env env, long repoId, BackupMeta backupMeta) { this(label, backupTs, dbId, dbName, jobInfo, allowLoad, replicaAlloc, timeoutMs, metaVersion, reserveReplica, - reserveDynamicPartitionEnable, isBeingSynced, isCleanTables, isCleanPartitions, isAtomicRestore, env, - repoId); + reserveDynamicPartitionEnable, isBeingSynced, isCleanTables, isCleanPartitions, isAtomicRestore, + storageResource, reserveStoragePolicy, env, repoId); this.backupMeta = backupMeta; } @@ -676,7 +680,8 @@ private void checkAndPrepareMeta() { } for (BackupJobInfo.BackupS3ResourceInfo backupS3ResourceInfo : jobInfo.newBackupObjects.s3Resources) { - Resource resource = Env.getCurrentEnv().getResourceMgr().getResource(backupS3ResourceInfo.name); + Resource resource = Env.getCurrentEnv().getResourceMgr().getResource(storageResource != null + ? storageResource : backupS3ResourceInfo.name); if (resource == null) { continue; } @@ -756,6 +761,16 @@ private void checkAndPrepareMeta() { BackupPartitionInfo backupPartInfo = partitionEntry.getValue(); Partition localPartition = localOlapTbl.getPartition(partitionName); Partition remotePartition = remoteOlapTbl.getPartition(partitionName); + + String policyName = remoteOlapTbl.getPartitionInfo() + .getDataProperty(remotePartition.getId()).getStoragePolicy(); + if (StringUtils.isNotEmpty(policyName)) { + status = new Status(ErrCode.COMMON_ERROR, "Can't restore remote partition " + + partitionName + " in table " + remoteTbl.getName() + " with storage policy " + + policyName + " when local table " + localTbl.getName() + " exist." + + " Please drop old table and restore again."); + return; + } if (localPartition != null) { // Partition already exist. PartitionInfo localPartInfo = localOlapTbl.getPartitionInfo(); @@ -841,7 +856,8 @@ private void checkAndPrepareMeta() { // reset all ids in this table String srcDbName = jobInfo.dbName; - Status st = remoteOlapTbl.resetIdsForRestore(env, db, replicaAlloc, reserveReplica, srcDbName); + Status st = remoteOlapTbl.resetIdsForRestore(env, db, replicaAlloc, + reserveReplica, reserveStoragePolicy, srcDbName); if (!st.ok()) { status = st; return; @@ -932,6 +948,18 @@ private void checkAndPrepareMeta() { if (isAtomicRestore && !restoredPartitions.isEmpty()) { throw new RuntimeException("atomic restore is set, but the restored partitions is not empty"); } + + // check and restore resources + checkAndRestoreResources(); + if (!status.ok()) { + return; + } + // check and restore storage policies, should before createReplicas to get storage_policy_id + checkAndRestoreStoragePolicies(); + if (!status.ok()) { + return; + } + for (Pair entry : restoredPartitions) { OlapTable localTbl = (OlapTable) db.getTableNullable(entry.first); Preconditions.checkNotNull(localTbl, localTbl.getName()); @@ -987,17 +1015,6 @@ private void checkAndPrepareMeta() { db.readUnlock(); } - // check and restore resources - checkAndRestoreResources(); - if (!status.ok()) { - return; - } - // check and restore storage policies - checkAndRestoreStoragePolicies(); - if (!status.ok()) { - return; - } - if (LOG.isDebugEnabled()) { LOG.debug("finished to restore resources. {}", this.jobId); } @@ -1303,54 +1320,87 @@ private void checkAndRestoreResources() { } } + if (!reserveStoragePolicy) { + return; + } + for (BackupJobInfo.BackupS3ResourceInfo backupS3ResourceInfo : jobInfo.newBackupObjects.s3Resources) { String backupResourceName = backupS3ResourceInfo.name; - Resource localResource = resourceMgr.getResource(backupResourceName); + Resource localResource = resourceMgr.getResource(storageResource != null + ? storageResource : backupResourceName); S3Resource remoteS3Resource = (S3Resource) backupMeta.getResource(backupResourceName); - if (localResource != null) { - if (localResource.getType() != Resource.ResourceType.S3) { - status = new Status(ErrCode.COMMON_ERROR, "The type of local resource " - + backupResourceName + " is not same as restored resource"); - return; - } - S3Resource localS3Resource = (S3Resource) localResource; - if (localS3Resource.getSignature(BackupHandler.SIGNATURE_VERSION) - != remoteS3Resource.getSignature(BackupHandler.SIGNATURE_VERSION)) { - status = new Status(ErrCode.COMMON_ERROR, "S3 resource " - + jobInfo.getAliasByOriginNameIfSet(backupResourceName) - + " already exist but with different properties"); + + if (storageResource != null) { + if (localResource != null) { + if (localResource.getType() != Resource.ResourceType.S3) { + status = new Status(ErrCode.COMMON_ERROR, "The type of local resource " + + backupResourceName + " is not same as restored resource"); + return; + } + S3Resource localS3Resource = (S3Resource) localResource; + if (localS3Resource.getProperty(S3Properties.ENDPOINT) + .equals(remoteS3Resource.getProperty(S3Properties.ENDPOINT)) + && localS3Resource.getProperty(S3Properties.BUCKET) + .equals(remoteS3Resource.getProperty(S3Properties.BUCKET)) + && localS3Resource.getProperty(S3Properties.ROOT_PATH) + .equals(remoteS3Resource.getProperty(S3Properties.ROOT_PATH))) { + status = new Status(ErrCode.COMMON_ERROR, "local S3 resource " + + storageResource + " root path " + localS3Resource.getProperty(S3Properties.ROOT_PATH) + + " should not same as restored resource root path"); + return; + } + } else { + status = new Status(ErrCode.COMMON_ERROR, + "The local resource " + storageResource + " is not exist."); return; } } else { - try { - // restore resource - resourceMgr.createResource(remoteS3Resource); - } catch (DdlException e) { - status = new Status(ErrCode.COMMON_ERROR, e.getMessage()); - return; + if (localResource != null) { + if (localResource.getType() != Resource.ResourceType.S3) { + status = new Status(ErrCode.COMMON_ERROR, "The type of local resource " + + backupResourceName + " is not same as restored resource"); + return; + } + S3Resource localS3Resource = (S3Resource) localResource; + if (localS3Resource.getSignature(BackupHandler.SIGNATURE_VERSION) + != remoteS3Resource.getSignature(BackupHandler.SIGNATURE_VERSION)) { + status = new Status(ErrCode.COMMON_ERROR, "S3 resource " + + jobInfo.getAliasByOriginNameIfSet(backupResourceName) + + " already exist but with different properties"); + return; + } + } else { + try { + // restore resource + resourceMgr.createResource(remoteS3Resource); + } catch (DdlException e) { + status = new Status(ErrCode.COMMON_ERROR, e.getMessage()); + return; + } + restoredResources.add(remoteS3Resource); } - restoredResources.add(remoteS3Resource); } } } private void checkAndRestoreStoragePolicies() { + if (!reserveStoragePolicy) { + return; + } PolicyMgr policyMgr = Env.getCurrentEnv().getPolicyMgr(); for (BackupJobInfo.BackupStoragePolicyInfo backupStoragePolicyInfo : jobInfo.newBackupObjects.storagePolicies) { String backupStoragePoliceName = backupStoragePolicyInfo.name; Optional localPolicy = policyMgr.findPolicy(backupStoragePoliceName, PolicyTypeEnum.STORAGE); StoragePolicy backupStoargePolicy = backupMeta.getStoragePolicy(backupStoragePoliceName); + + // use specified storageResource + if (storageResource != null) { + backupStoargePolicy.setStorageResource(storageResource); + } if (localPolicy.isPresent()) { StoragePolicy localStoargePolicy = (StoragePolicy) localPolicy.get(); - if (localStoargePolicy.getVersion() - != backupStoargePolicy.getVersion()) { - status = new Status(ErrCode.COMMON_ERROR, "Storage policy " - + jobInfo.getAliasByOriginNameIfSet(backupStoragePoliceName) - + " already exist but with different properties"); - return; - } - // storage policy id should be same + // storage policy name and resource name should be same if (localStoargePolicy.getSignature(BackupHandler.SIGNATURE_VERSION) != backupStoargePolicy.getSignature(BackupHandler.SIGNATURE_VERSION)) { status = new Status(ErrCode.COMMON_ERROR, "Storage policy " @@ -1447,7 +1497,8 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc localTbl.getPartitionInfo().getTabletType(restorePart.getId()), null, localTbl.getCompressionType(), - localTbl.getEnableUniqueKeyMergeOnWrite(), localTbl.getStoragePolicy(), + localTbl.getEnableUniqueKeyMergeOnWrite(), + storagePolicy, localTbl.disableAutoCompaction(), localTbl.enableSingleReplicaCompaction(), localTbl.skipWriteIndexOnLoad(), @@ -1662,6 +1713,22 @@ private void replayCheckAndPrepareMeta() { } } + // restored resource + ResourceMgr resourceMgr = Env.getCurrentEnv().getResourceMgr(); + for (Resource resource : restoredResources) { + resourceMgr.replayCreateResource(resource); + } + + // restored storage policy + PolicyMgr policyMgr = Env.getCurrentEnv().getPolicyMgr(); + for (StoragePolicy storagePolicy : storagePolicies) { + Optional localPolicy = policyMgr.findPolicy(storagePolicy.getPolicyName(), + PolicyTypeEnum.STORAGE); + if (!localPolicy.isPresent()) { + policyMgr.replayCreate(storagePolicy); + } + } + // restored partitions for (Pair entry : restoredPartitions) { OlapTable localTbl = (OlapTable) db.getTableNullable(entry.first); @@ -1733,22 +1800,6 @@ private void replayCheckAndPrepareMeta() { } } - // restored resource - ResourceMgr resourceMgr = Env.getCurrentEnv().getResourceMgr(); - for (Resource resource : restoredResources) { - resourceMgr.replayCreateResource(resource); - } - - // restored storage policy - PolicyMgr policyMgr = Env.getCurrentEnv().getPolicyMgr(); - for (StoragePolicy storagePolicy : storagePolicies) { - Optional localPolicy = policyMgr.findPolicy(storagePolicy.getPolicyName(), - PolicyTypeEnum.STORAGE); - if (!localPolicy.isPresent()) { - policyMgr.replayCreate(storagePolicy); - } - } - LOG.info("replay check and prepare meta. {}", this); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 840311028ead72..ed7dbe3376a912 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -769,7 +769,7 @@ public void resetVersionForRestore() { } public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restoreReplicaAlloc, - boolean reserveReplica, String srcDbName) { + boolean reserveReplica, boolean reserveStoragePolicy, String srcDbName) { // ATTN: The meta of the restore may come from different clusters, so the // original ID in the meta may conflict with the ID of the new cluster. For // example, if a newly allocated ID happens to be the same as an original ID, @@ -818,7 +818,7 @@ public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restore boolean isSinglePartition = partitionInfo.getType() != PartitionType.RANGE && partitionInfo.getType() != PartitionType.LIST; partitionInfo.resetPartitionIdForRestore(partitionMap, - reserveReplica ? null : restoreReplicaAlloc, isSinglePartition); + reserveReplica ? null : restoreReplicaAlloc, reserveStoragePolicy, isSinglePartition); // for each partition, reset rollup index map Map nextIndexes = Maps.newHashMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index 304a105b8cf92d..a54d7c83cbf606 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -416,7 +416,7 @@ public void moveFromTempToFormal(long tempPartitionId) { public void resetPartitionIdForRestore( Map partitionIdMap, - ReplicaAllocation restoreReplicaAlloc, boolean isSinglePartitioned) { + ReplicaAllocation restoreReplicaAlloc, boolean reserveStoragePolicy, boolean isSinglePartitioned) { Map origIdToDataProperty = idToDataProperty; Map origIdToReplicaAllocation = idToReplicaAllocation; Map origIdToItem = idToItem; @@ -429,7 +429,8 @@ public void resetPartitionIdForRestore( idToStoragePolicy = Maps.newHashMap(); for (Map.Entry entry : partitionIdMap.entrySet()) { - idToDataProperty.put(entry.getKey(), origIdToDataProperty.get(entry.getValue())); + idToDataProperty.put(entry.getKey(), reserveStoragePolicy + ? origIdToDataProperty.get(entry.getValue()) : DataProperty.DEFAULT_HDD_DATA_PROPERTY); idToReplicaAllocation.put(entry.getKey(), restoreReplicaAlloc == null ? origIdToReplicaAllocation.get(entry.getValue()) : restoreReplicaAlloc); @@ -437,7 +438,8 @@ public void resetPartitionIdForRestore( idToItem.put(entry.getKey(), origIdToItem.get(entry.getValue())); } idToInMemory.put(entry.getKey(), origIdToInMemory.get(entry.getValue())); - idToStoragePolicy.put(entry.getKey(), origIdToStoragePolicy.get(entry.getValue())); + idToStoragePolicy.put(entry.getKey(), reserveStoragePolicy + ? origIdToStoragePolicy.get(entry.getValue()) : ""); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java index 6d3c8e7f797d3e..87bfed4c2252e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/StoragePolicy.java @@ -419,6 +419,7 @@ public int getSignature(int signatureVersion) { adler32.update(signatureVersion); final String charsetName = "UTF-8"; + //ignore check id, version, cooldownTimestampMs, cooldownTtl try { // policy name adler32.update(policyName.getBytes(charsetName)); @@ -426,35 +427,15 @@ public int getSignature(int signatureVersion) { LOG.debug("signature. policy name: {}", policyName); } // storageResource name - adler32.update(policyName.getBytes(charsetName)); + adler32.update(storageResource.getBytes(charsetName)); if (LOG.isDebugEnabled()) { LOG.debug("signature. storageResource name: {}", storageResource); } - // id - adler32.update(String.valueOf(id).getBytes(charsetName)); - if (LOG.isDebugEnabled()) { - LOG.debug("signature. id : {}", id); - } // type adler32.update(String.valueOf(getType()).getBytes(charsetName)); if (LOG.isDebugEnabled()) { LOG.debug("signature. type : {}", getType()); } - // version - adler32.update(String.valueOf(getVersion()).getBytes(charsetName)); - if (LOG.isDebugEnabled()) { - LOG.debug("signature. version : {}", getVersion()); - } - // cooldownTimestampMs - adler32.update(String.valueOf(cooldownTimestampMs).getBytes(charsetName)); - if (LOG.isDebugEnabled()) { - LOG.debug("signature. cooldownTimestampMs : {}", cooldownTimestampMs); - } - // cooldownTtl - adler32.update(String.valueOf(cooldownTtl).getBytes(charsetName)); - if (LOG.isDebugEnabled()) { - LOG.debug("signature. cooldownTtl : {}", cooldownTtl); - } } catch (UnsupportedEncodingException e) { LOG.error("encoding error", e); return -1; diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java index c5473b7739b255..4fd6ccb1643764 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java @@ -257,7 +257,7 @@ boolean await(long timeout, TimeUnit unit) { db.unregisterTable(expectedRestoreTbl.getName()); job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(), db.getFullName(), jobInfo, false, - new ReplicaAllocation((short) 3), 100000, -1, false, false, false, false, false, false, + new ReplicaAllocation((short) 3), 100000, -1, false, false, false, false, false, false, null, false, env, repo.getId()); List
tbls = Lists.newArrayList(); diff --git a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy index ea503a8cbaf539..0585cf9b587570 100644 --- a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy +++ b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy @@ -235,4 +235,1156 @@ suite("test_backup_cooldown", "backup_cooldown_data") { sql """ drop resource ${resource_name2}; """ +} + +// test restore back to old instance +suite("test_backup_cooldown_1", "backup_cooldown_data") { + + String suiteName = "test_backup_cooldown_1" + String resource_name1 = "resource_${suiteName}_1" + String policy_name1 = "policy_${suiteName}_1" + String resource_name2 = "resource_${suiteName}_2" + String policy_name2 = "policy_${suiteName}_2" + String dbName = "${suiteName}_db" + String tableName = "${suiteName}_table" + String snapshotName = "${suiteName}_snapshot" + String repoName = "${suiteName}_repo" + def found = 0 + def records + def result + def row + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + + + + sql """ + CREATE RESOURCE IF NOT EXISTS "${resource_name1}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "AWS_REGION" = "${getS3Region()}", + "AWS_ROOT_PATH" = "regression/cooldown1", + "AWS_ACCESS_KEY" = "${getS3AK()}", + "AWS_SECRET_KEY" = "${getS3SK()}", + "AWS_MAX_CONNECTIONS" = "50", + "AWS_REQUEST_TIMEOUT_MS" = "3000", + "AWS_CONNECTION_TIMEOUT_MS" = "1000", + "AWS_BUCKET" = "${getS3BucketName()}", + "s3_validity_check" = "true" + ); + """ + + sql """ + CREATE RESOURCE IF NOT EXISTS "${resource_name2}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "AWS_REGION" = "${getS3Region()}", + "AWS_ROOT_PATH" = "regression/cooldown2", + "AWS_ACCESS_KEY" = "${getS3AK()}", + "AWS_SECRET_KEY" = "${getS3SK()}", + "AWS_MAX_CONNECTIONS" = "50", + "AWS_REQUEST_TIMEOUT_MS" = "3000", + "AWS_CONNECTION_TIMEOUT_MS" = "1000", + "AWS_BUCKET" = "${getS3BucketName()}", + "s3_validity_check" = "true" + ); + """ + + sql """ + CREATE STORAGE POLICY IF NOT EXISTS ${policy_name1} + PROPERTIES( + "storage_resource" = "${resource_name1}", + "cooldown_ttl" = "10" + ) + """ + + sql """ + CREATE STORAGE POLICY IF NOT EXISTS ${policy_name2} + PROPERTIES( + "storage_resource" = "${resource_name2}", + "cooldown_ttl" = "10" + ) + """ + + //generate_cooldown_task_interval_sec default is 20 + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + + sql """ + CREATE TABLE ${dbName}.${tableName} + ( + k1 BIGINT, + v1 VARCHAR(48), + INDEX idx1 (v1) USING INVERTED PROPERTIES("parser" = "english") + ) + DUPLICATE KEY(k1) + PARTITION BY RANGE(`k1`) + ( + PARTITION p201701 VALUES [(0), (3)) ("storage_policy" = "${policy_name1}"), + PARTITION `p201702` VALUES LESS THAN (6)("storage_policy" = "${policy_name2}"), + PARTITION `p2018` VALUES [(6),(100)) + ) + DISTRIBUTED BY HASH (k1) BUCKETS 3 + PROPERTIES( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + List values = [] + for (int i = 1; i <= 10; ++i) { + values.add("(${i}, ${i})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + + // // wait cooldown + // result = sql "show data FROM ${dbName}.${tableName}" + // sqlResult = result[0][5].toString(); + // int count = 0; + // while (sqlResult.contains("0.00")) { + // if (++count >= 120) { // 10min + // logger.error('cooldown task is timeouted') + // throw new Exception("cooldown task is timeouted after 10 mins") + // } + // Thread.sleep(5000) + + // result = sql "show data FROM ${dbName}.${tableName}" + // sqlResult = result[0][5].toString(); + + + // } + + // assertNotEquals('0.000 ', result[0][5].toString()) + + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + syncer.waitSnapshotFinish(dbName) + + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + // 1 老表存在的情况 + // 1.1 restore 不指定, 预期成功，且落冷 + // 1.2 restore 指定 （"reserve_storage_policy"="true"）， 预期成功，且落冷 + // 1.3 restore 指定 （"reserve_storage_policy"="false"），预期成功，且不落冷 + + // 2 删除表 + // 1.1 restore 不指定 预期成功，且落冷 + // 1.2 restore 指定 （"reserve_storage_policy"="true"）预期成功，且落冷 + // 1.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 + + + // 3 删除resource 和 policy + // 1.1 restore 不指定 预期成功，且落冷 + // 1.2 restore 指定 （"reserve_storage_policy"="true"）预期成功，且落冷 + // 1.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 + + + // 1. old table exist + // 1.1 restore normal + // 1.2 restore with（"reserve_storage_policy"="true"） + // 1.3 restore with（"reserve_storage_policy"="false"） + logger.info(" ====================================== 1.1 ==================================== ") + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + // restore failed + records = sql_return_maparray "SHOW restore FROM ${dbName}" + row = records[records.size() - 1] + assertTrue(row.Status.contains("Can't restore remote partition")) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + + logger.info(" ====================================== 1.2 ==================================== ") + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "reserve_storage_policy"="true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + // restore failed + records = sql_return_maparray "SHOW restore FROM ${dbName}" + row = records[records.size() - 1] + assertTrue(row.Status.contains("Can't restore remote partition")) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + + logger.info(" ====================================== 1.3 ==================================== ") + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "reserve_storage_policy"="false" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + // restore failed + records = sql_return_maparray "SHOW restore FROM ${dbName}" + row = records[records.size() - 1] + assertTrue(row.Status.contains("Can't restore remote partition")) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + + // 2. drop old table + // 2.1 restore normal + // 2.2 restore with （"reserve_storage_policy"="true"） + // 2.3 restore with （"reserve_storage_policy"="false"） + logger.info(" ====================================== 2.1 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) + + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + } + assertNotEquals('0.000 ', result[0][5].toString()) + + + logger.info(" ====================================== 2.2 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "reserve_storage_policy"="true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) + + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + } + assertNotEquals('0.000 ', result[0][5].toString()) + + + logger.info(" ====================================== 2.3 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "reserve_storage_policy"="false" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // check table don't have storage_policy + records = sql_return_maparray "show storage policy using" + found = 0 + for (def res2 : records) { + if (res2.Database.equals(dbName) && res2.Table.equals(tableName)) { + found = 1 + break + } + } + assertEquals(found, 0) + + + // 3. drop old table and resource and policy + // 3.1 restore normal + // 3.2 restore with（"reserve_storage_policy"="true"） + // 3.3 restore with（"reserve_storage_policy"="false"） + logger.info(" ====================================== 3.1 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop resource ${resource_name1}; + """ + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) + + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + } + assertNotEquals('0.000 ', result[0][5].toString()) + + + logger.info(" ====================================== 3.2 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop resource ${resource_name1}; + """ + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "reserve_storage_policy"="true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) + + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + } + assertNotEquals('0.000 ', result[0][5].toString()) + + + logger.info(" ====================================== 3.3 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop resource ${resource_name1}; + """ + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "reserve_storage_policy"="false" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // check table don't have storage_policy + records = sql_return_maparray "show storage policy using" + found = 0 + for (def res2 : records) { + if (res2.Database.equals(dbName) && res2.Table.equals(tableName)) { + found = 1 + break + } + } + assertEquals(found, 0) + + // check storage policy ${policy_name1} not exist + records = sql_return_maparray "show storage policy" + found = 0 + for (def res2 : records) { + if (res2.PolicyName.equals(policy_name1)) { + found = 1 + break + } + } + assertEquals(found, 0) + + // check resource ${resource_name1} not exist + records = sql_return_maparray "show storage policy" + found = 0 + for (def res2 : records) { + if (res2.Name.equals(resource_name1)) { + found = 1 + break + } + } + assertEquals(found, 0) + + + // 4. alter policy + logger.info(" ====================================== 4.1 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + ALTER STORAGE POLICY ${policy_name2} PROPERTIES ("cooldown_ttl" = "11"); + """ + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) + + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + } + assertNotEquals('0.000 ', result[0][5].toString()) + + // check storage policy ${policy_name2} exist + records = sql_return_maparray "show storage policy" + found = 0 + for (def res2 : records) { + if (res2.PolicyName.equals(policy_name2) && res2.CooldownTtl.equals("11")) { + found = 1 + break + } + } + assertEquals(found, 1) + + + + //cleanup + sql "DROP TABLE IF EXISTS ${dbName}.${tableName} FORCE" + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP REPOSITORY `${repoName}`" + + sql """ + drop storage policy ${policy_name1}; + """ + + sql """ + drop resource ${resource_name1}; + """ + + sql """ + drop storage policy ${policy_name2}; + """ + + sql """ + drop resource ${resource_name2}; + """ +} + + + + + +// test restore back to a new instance +suite("test_backup_cooldown_2", "backup_cooldown_data") { + + String suiteName = "test_backup_cooldown_2" + String resource_name1 = "resource_${suiteName}_1" + String policy_name1 = "policy_${suiteName}_1" + String resource_name2 = "resource_${suiteName}_2" + String resource_new_name = "resource_${suiteName}_new" + String policy_name2 = "policy_${suiteName}_2" + String dbName = "${suiteName}_db" + String tableName = "${suiteName}_table" + String snapshotName = "${suiteName}_snapshot" + String repoName = "${suiteName}_repo" + def found = 0 + def records + def syncer = getSyncer() + def result + syncer.createS3Repository(repoName) + + + + sql """ + CREATE RESOURCE IF NOT EXISTS "${resource_name1}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "AWS_REGION" = "${getS3Region()}", + "AWS_ROOT_PATH" = "regression/cooldown1", + "AWS_ACCESS_KEY" = "${getS3AK()}", + "AWS_SECRET_KEY" = "${getS3SK()}", + "AWS_MAX_CONNECTIONS" = "50", + "AWS_REQUEST_TIMEOUT_MS" = "3000", + "AWS_CONNECTION_TIMEOUT_MS" = "1000", + "AWS_BUCKET" = "${getS3BucketName()}", + "s3_validity_check" = "true" + ); + """ + + sql """ + CREATE RESOURCE IF NOT EXISTS "${resource_name2}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "AWS_REGION" = "${getS3Region()}", + "AWS_ROOT_PATH" = "regression/cooldown2", + "AWS_ACCESS_KEY" = "${getS3AK()}", + "AWS_SECRET_KEY" = "${getS3SK()}", + "AWS_MAX_CONNECTIONS" = "50", + "AWS_REQUEST_TIMEOUT_MS" = "3000", + "AWS_CONNECTION_TIMEOUT_MS" = "1000", + "AWS_BUCKET" = "${getS3BucketName()}", + "s3_validity_check" = "true" + ); + """ + + sql """ + CREATE RESOURCE IF NOT EXISTS "${resource_new_name}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "AWS_REGION" = "${getS3Region()}", + "AWS_ROOT_PATH" = "regression/cooldown3", + "AWS_ACCESS_KEY" = "${getS3AK()}", + "AWS_SECRET_KEY" = "${getS3SK()}", + "AWS_MAX_CONNECTIONS" = "50", + "AWS_REQUEST_TIMEOUT_MS" = "3000", + "AWS_CONNECTION_TIMEOUT_MS" = "1000", + "AWS_BUCKET" = "${getS3BucketName()}", + "s3_validity_check" = "true" + ); + """ + + sql """ + CREATE STORAGE POLICY IF NOT EXISTS ${policy_name1} + PROPERTIES( + "storage_resource" = "${resource_name1}", + "cooldown_ttl" = "10" + ) + """ + + sql """ + CREATE STORAGE POLICY IF NOT EXISTS ${policy_name2} + PROPERTIES( + "storage_resource" = "${resource_name2}", + "cooldown_ttl" = "10" + ) + """ + + //generate_cooldown_task_interval_sec default is 20 + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + + sql """ + CREATE TABLE ${dbName}.${tableName} + ( + k1 BIGINT, + v1 VARCHAR(48), + INDEX idx1 (v1) USING INVERTED PROPERTIES("parser" = "english") + ) + DUPLICATE KEY(k1) + PARTITION BY RANGE(`k1`) + ( + PARTITION p201701 VALUES [(0), (3)) ("storage_policy" = "${policy_name1}"), + PARTITION `p201702` VALUES LESS THAN (6)("storage_policy" = "${policy_name2}"), + PARTITION `p2018` VALUES [(6),(100)) + ) + DISTRIBUTED BY HASH (k1) BUCKETS 3 + PROPERTIES( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + List values = [] + for (int i = 1; i <= 10; ++i) { + values.add("(${i}, ${i})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + int count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) + + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + } + assertNotEquals('0.000 ', result[0][5].toString()) + + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + syncer.waitSnapshotFinish(dbName) + + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + // 1 老表存在的情况 + // 1.1 restore 指定 （"storage_resource"="resource_name1"）， 预期失败，路径需不一致 + // 1.2 restore 指定 （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 + // 1.3 restore 指定 （"storage_resource"="resource_new_name"）， 预期失败，有stroage policy属性的表不允许restore到已存在的表 + + + // 2 删除表 + // 2.1 restore 指定 （"storage_resource"="resource_name1"）， 预期失败，resource路径需不一致 + // 2.2 restore 指定 （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 + // 2.3 restore 指定 （"storage_resource"="resource_new_name"）， storage policy 存在失败 + + + // 3 删除表和policy + // 3.1 restore 指定 （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 + // 3.2 restore 指定 （"storage_resource"="resource_new_name"）， 成功 + + + + // 4 删除表和policy 同时指定storage_resource和reserve_storage_policy + // 4.1 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="true"），预期失败，resource不存在 + // 4.2 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="false"），预期失败，resource不存在 + // 4.3 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="true"），预期成功 + // 4.4 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="false"），预期成功 + + + + // 1 old table exist + // 1.1 restore with （"storage_resource"="resource_name1"）， 预期失败，路径需不一致 + // 1.2 restore with （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 + // 1.3 restore with （"storage_resource"="resource_new_name"）， 预期失败，有stroage policy属性的表不允许restore到已存在的表 + logger.info(" ====================================== 1.1 ==================================== ") + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="${resource_name1}" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // restore failed + records = sql_return_maparray "SHOW restore FROM ${dbName}" + row = records[records.size() - 1] + assertTrue(row.Status.contains("Can't restore remote partition")) + + + + + logger.info(" ====================================== 1.2 ==================================== ") + def fail_restore_1 = try_sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="resource_name_not_exist" + ) + """ + + logger.info("fail_restore_1: ${fail_restore_1}") + + assertEquals(fail_restore_1, null) + + // 1.3 restore 指定 （"storage_resource"="resource_new_name"）， 预期失败，有stroage policy属性的表不允许restore到已存在的表 + logger.info(" ====================================== 1.3 ==================================== ") + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="${resource_new_name}" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // restore failed + records = sql_return_maparray "SHOW restore FROM ${dbName}" + row = records[records.size() - 1] + assertTrue(row.Status.contains("Can't restore remote partition")) + + + + + // 2 删除表 + // 2.1 restore 指定 （"storage_resource"="resource_name1"）， 预期失败，resource路径需不一致 + // 2.2 restore 指定 （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 + // 2.3 restore 指定 （"storage_resource"="resource_new_name"）， storage policy 存在失败 + logger.info(" ====================================== 2.1 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="${resource_name1}" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + // restore failed + records = sql_return_maparray "SHOW restore FROM ${dbName}" + row = records[records.size() - 1] + assertTrue(row.Status.contains("should not same as restored resource root path")) + + + + + logger.info(" ====================================== 2.2 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + def fail_restore_2 = try_sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="resource_name_not_exist" + ) + """ + + assertEquals(fail_restore_2, null) + + + // 2.3 restore 指定 （"storage_resource"="resource_new_name"）， storage policy 存在失败 + logger.info(" ====================================== 2.3 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="${resource_new_name}" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + // restore failed + records = sql_return_maparray "SHOW restore FROM ${dbName}" + row = records[records.size() - 1] + assertTrue(row.Status.contains("already exist but with different properties")) + + + // 3 删除表和policy + // 3.1 restore 指定 （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 + // 3.2 restore 指定 （"storage_resource"="resource_new_name"）， 成功 + logger.info(" ====================================== 3.1 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop storage policy ${policy_name2}; + """ + def fail_restore_3 = try_sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="resource_name_not_exist" + ) + """ + + assertEquals(fail_restore_3, null) + + logger.info(" ====================================== 3.2 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop storage policy ${policy_name2}; + """ + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="${resource_new_name}" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) + + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + } + assertNotEquals('0.000 ', result[0][5].toString()) + + // check plocy_name1 storage_resource change to resource_new_name + records = sql_return_maparray "show storage policy" + found = 0 + for (def res2 : records) { + if (res2.StorageResource.equals(resource_new_name) && res2.PolicyName.equals(policy_name1)) { + found = 1 + break + } + } + assertEquals(found, 1) + + // check plocy_name2 storage_resource change to resource_new_name + records = sql_return_maparray "show storage policy" + found = 0 + for (def res2 : records) { + if (res2.StorageResource.equals(resource_new_name) && res2.PolicyName.equals(policy_name2)) { + found = 1 + break + } + } + assertEquals(found, 1) + + + + // 4 删除表和policy + // 4.1 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="true"），预期失败，resource不存在 + // 4.2 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="false"），预期失败，resource不存在 + // 4.3 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="true"），预期成功 + // 4.4 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="false"），预期成功 + logger.info(" ====================================== 4.1 ==================================== ") + sql "DROP TABLE if exists ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop storage policy ${policy_name2}; + """ + def fail_restore_4 = try_sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="resource_name_not_exist", + "reserve_storage_policy"="true" + ) + """ + + assertEquals(fail_restore_4, null) + + + // 4.2 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="false"），预期成功 + logger.info(" ====================================== 4.2 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop storage policy ${policy_name2}; + """ + + def fail_restore_5 = try_sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="resource_name_not_exist", + "reserve_storage_policy"="false" + ) + """ + + assertEquals(fail_restore_5, null) + + + // 4.3 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="true"），预期成功 + logger.info(" ====================================== 4.3 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop storage policy ${policy_name2}; + """ + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="${resource_new_name}", + "reserve_storage_policy"="true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // check plocy_name1 storage_resource change to resource_new_name + records = sql_return_maparray "show storage policy" + found = 0 + for (def res2 : records) { + if (res2.StorageResource.equals(resource_new_name) && res2.PolicyName.equals(policy_name1)) { + found = 1 + break + } + } + assertEquals(found, 1) + + // check plocy_name2 storage_resource change to resource_new_name + records = sql_return_maparray "show storage policy" + found = 0 + for (def res2 : records) { + if (res2.StorageResource.equals(resource_new_name) && res2.PolicyName.equals(policy_name2)) { + found = 1 + break + } + } + assertEquals(found, 1) + + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) + + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + } + assertNotEquals('0.000 ', result[0][5].toString()) + + + + // 4.4 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="false"），预期成功 + logger.info(" ====================================== 4.4 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop storage policy ${policy_name2}; + """ + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "storage_resource"="${resource_new_name}", + "reserve_storage_policy"="false" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + + // check table don't have storage_policy + records = sql_return_maparray "show storage policy using" + found = 0 + for (def res2 : records) { + if (res2.Database.equals(dbName) && res2.Table.equals(tableName)) { + found = 1 + break + } + } + assertEquals(found, 0) + + + //cleanup + sql "DROP TABLE IF EXISTS ${dbName}.${tableName} FORCE" + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP REPOSITORY `${repoName}`" + + try_sql """ + drop storage policy ${policy_name1}; + """ + + try_sql """ + drop resource ${resource_name1}; + """ + + try_sql """ + drop storage policy ${policy_name2}; + """ + + try_sql """ + drop resource ${resource_name2}; + """ } \ No newline at end of file From f6cd07798c85b896cfed8e1857b9c91e9201dce5 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Mon, 11 Nov 2024 14:29:13 +0800 Subject: [PATCH 05/15] fix tmp --- .../apache/doris/analysis/RestoreStmt.java | 8 ++-- .../org/apache/doris/backup/BackupJob.java | 16 ++------ .../org/apache/doris/backup/RestoreJob.java | 41 +++++++++---------- 3 files changed, 28 insertions(+), 37 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java index 2287ea55be9b50..655b5fb017d8da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java @@ -46,7 +46,7 @@ public class RestoreStmt extends AbstractBackupStmt implements NotFallbackInPars public static final String PROP_CLEAN_TABLES = "clean_tables"; public static final String PROP_CLEAN_PARTITIONS = "clean_partitions"; public static final String PROP_ATOMIC_RESTORE = "atomic_restore"; - public static final String PROP_STORAGE_SOURCE = "storage_resource"; + public static final String PROP_STORAGE_RESOURCE = "storage_resource"; public static final String PROP_RESERVE_STORAGE_POLICY = "reserve_storage_policy"; private boolean allowLoad = false; @@ -226,8 +226,8 @@ public void analyzeProperties() throws AnalysisException { // is atomic restore isAtomicRestore = eatBooleanProperty(copiedProperties, PROP_ATOMIC_RESTORE, isAtomicRestore); - if (copiedProperties.containsKey(PROP_STORAGE_SOURCE)) { - storageResource = copiedProperties.get(PROP_STORAGE_SOURCE); + if (copiedProperties.containsKey(PROP_STORAGE_RESOURCE)) { + storageResource = copiedProperties.get(PROP_STORAGE_RESOURCE); Resource localResource = Env.getCurrentEnv().getResourceMgr().getResource(storageResource); if (localResource == null) { @@ -241,7 +241,7 @@ public void analyzeProperties() throws AnalysisException { + storageResource + " is not same as restored resource"); } - copiedProperties.remove(PROP_STORAGE_SOURCE); + copiedProperties.remove(PROP_STORAGE_RESOURCE); } // reserve storage policy diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 4189851a7c0c91..f2c910341c931b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -497,7 +497,6 @@ private void prepareAndSendSnapshotTask() { // copy all related schema at this moment List
copiedTables = Lists.newArrayList(); List copiedResources = Lists.newArrayList(); - AgentBatchTask batchTask = new AgentBatchTask(Config.backup_restore_batch_task_num_per_rpc); List copiedStoragePolicys = Lists.newArrayList(); AgentBatchTask batchTask = new AgentBatchTask(Config.backup_restore_batch_task_num_per_rpc); for (TableRef tableRef : tableRefs) { @@ -516,15 +515,10 @@ private void prepareAndSendSnapshotTask() { OlapTable olapTable = (OlapTable) tbl; checkOlapTable(olapTable, tableRef); if (getContent() == BackupContent.ALL) { - if (!prepareSnapshotTaskForOlapTableWithoutLock( - db, (OlapTable) tbl, tableRef, batchTask).ok()) { - return; - } - } - if (!prepareBackupMetaForOlapTableWithoutLock(tableRef, olapTable, copiedTables, - copiedStoragePolicys).ok()) { - return; + prepareSnapshotTaskForOlapTableWithoutLock(db, (OlapTable) tbl, tableRef, batchTask); } + prepareBackupMetaForOlapTableWithoutLock(tableRef, olapTable, copiedTables, + copiedStoragePolicys); break; case VIEW: prepareBackupMetaForViewWithoutLock((View) tbl, copiedTables); @@ -676,7 +670,7 @@ private void checkResourceForOdbcTable(OdbcTable odbcTable) { } } - private Status prepareBackupMetaForOlapTableWithoutLock(TableRef tableRef, OlapTable olapTable, + private void prepareBackupMetaForOlapTableWithoutLock(TableRef tableRef, OlapTable olapTable, List
copiedTables, List copiedStoragePolicys) { // only copy visible indexes @@ -707,8 +701,6 @@ private Status prepareBackupMetaForOlapTableWithoutLock(TableRef tableRef, OlapT copiedStoragePolicys.add(storagePolicy); } } - - return Status.OK; } private void prepareBackupMetaForViewWithoutLock(View view, List
copiedTables) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 6f5b80b7c56370..441fc1c7a3cd9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -69,15 +69,13 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.property.S3ClientBEProperties; -<<<<<<< HEAD +import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; -======= import org.apache.doris.policy.Policy; import org.apache.doris.policy.PolicyMgr; import org.apache.doris.policy.PolicyTypeEnum; import org.apache.doris.policy.StoragePolicy; ->>>>>>> 9bd9133568... [feature](cooldown)backup cooldown data import org.apache.doris.resource.Tag; import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTask; @@ -108,6 +106,7 @@ import com.google.common.collect.Table.Cell; import com.google.gson.annotations.SerializedName; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -132,6 +131,8 @@ public class RestoreJob extends AbstractJob implements GsonPostProcessable { private static final String PROP_CLEAN_TABLES = RestoreStmt.PROP_CLEAN_TABLES; private static final String PROP_CLEAN_PARTITIONS = RestoreStmt.PROP_CLEAN_PARTITIONS; private static final String PROP_ATOMIC_RESTORE = RestoreStmt.PROP_ATOMIC_RESTORE; + private static final String PROP_STORAGE_RESOURCE = RestoreStmt.PROP_STORAGE_RESOURCE; + private static final String PROP_RESERVE_STORAGE_POLICY = RestoreStmt.PROP_RESERVE_STORAGE_POLICY; private static final String ATOMIC_RESTORE_TABLE_PREFIX = "__doris_atomic_restore_prefix__"; private static final Logger LOG = LogManager.getLogger(RestoreJob.class); @@ -191,6 +192,7 @@ public enum RestoreJobState { private List
restoredTbls = Lists.newArrayList(); @SerializedName("rr") private List restoredResources = Lists.newArrayList(); + @SerializedName("sp") private List storagePolicies = Lists.newArrayList(); // save all restored partitions' version info which are already exist in catalog @@ -220,7 +222,10 @@ public enum RestoreJobState { private boolean isCleanPartitions = false; // Whether to restore the data into a temp table, and then replace the origin one. private boolean isAtomicRestore = false; - + // the target storage resource + private String storageResource = null; + // whether to reserve storage policy + private boolean reserveStoragePolicy = false; // restore properties @SerializedName("prop") private Map properties = Maps.newHashMap(); @@ -265,6 +270,8 @@ public RestoreJob(String label, String backupTs, long dbId, String dbName, Backu properties.put(PROP_CLEAN_TABLES, String.valueOf(isCleanTables)); properties.put(PROP_CLEAN_PARTITIONS, String.valueOf(isCleanPartitions)); properties.put(PROP_ATOMIC_RESTORE, String.valueOf(isAtomicRestore)); + properties.put(PROP_STORAGE_RESOURCE, String.valueOf(storageResource)); + properties.put(PROP_RESERVE_STORAGE_POLICY, String.valueOf(reserveStoragePolicy)); } public RestoreJob(String label, String backupTs, long dbId, String dbName, BackupJobInfo jobInfo, boolean allowLoad, @@ -1485,6 +1492,11 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc Env.getCurrentInvertedIndex().addTablet(restoreTablet.getId(), tabletMeta); for (Replica restoreReplica : restoreTablet.getReplicas()) { Env.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica); + String storagePolicy = ""; + if (reserveStoragePolicy) { + storagePolicy = localTbl.getPartitionInfo() + .getDataProperty(restorePart.getId()).getStoragePolicy(); + } CreateReplicaTask task = new CreateReplicaTask(restoreReplica.getBackendIdWithoutException(), dbId, localTbl.getId(), restorePart.getId(), restoredIdx.getId(), restoreTablet.getId(), restoreReplica.getId(), indexMeta.getShortKeyColumnCount(), @@ -2744,23 +2756,6 @@ public void readFields(DataInput in) throws IOException { } else { readOthers(in); } - - out.writeInt(restoredResources.size()); - for (Resource resource : restoredResources) { - resource.write(out); - } - - out.writeInt(storagePolicies.size()); - for (StoragePolicy policy : storagePolicies) { - policy.write(out); - } - - // write properties - out.writeInt(properties.size()); - for (Map.Entry entry : properties.entrySet()) { - Text.writeString(out, entry.getKey()); - Text.writeString(out, entry.getValue()); - } } private void readOthers(DataInput in) throws IOException { @@ -2848,6 +2843,8 @@ private void readOthers(DataInput in) throws IOException { isCleanTables = Boolean.parseBoolean(properties.get(PROP_CLEAN_TABLES)); isCleanPartitions = Boolean.parseBoolean(properties.get(PROP_CLEAN_PARTITIONS)); isAtomicRestore = Boolean.parseBoolean(properties.get(PROP_ATOMIC_RESTORE)); + storageResource = properties.get(PROP_STORAGE_RESOURCE); + reserveStoragePolicy = Boolean.parseBoolean(properties.get(PROP_RESERVE_STORAGE_POLICY)); } @Override @@ -2858,6 +2855,8 @@ public void gsonPostProcess() throws IOException { isCleanTables = Boolean.parseBoolean(properties.get(PROP_CLEAN_TABLES)); isCleanPartitions = Boolean.parseBoolean(properties.get(PROP_CLEAN_PARTITIONS)); isAtomicRestore = Boolean.parseBoolean(properties.get(PROP_ATOMIC_RESTORE)); + storageResource = properties.get(PROP_STORAGE_RESOURCE); + reserveStoragePolicy = Boolean.parseBoolean(properties.get(PROP_RESERVE_STORAGE_POLICY)); } @Override From afe66cc473e11dad72060b0fff3d194d7b23bcc1 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Mon, 11 Nov 2024 23:27:29 +0800 Subject: [PATCH 06/15] update test_backup_restore_cold_data.groovy --- .../test_backup_restore_cold_data.groovy | 120 +++++++++--------- 1 file changed, 57 insertions(+), 63 deletions(-) diff --git a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy index 0585cf9b587570..11d890a20ef019 100644 --- a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy +++ b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy @@ -343,24 +343,22 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { assertEquals(result.size(), values.size()); - // // wait cooldown - // result = sql "show data FROM ${dbName}.${tableName}" - // sqlResult = result[0][5].toString(); - // int count = 0; - // while (sqlResult.contains("0.00")) { - // if (++count >= 120) { // 10min - // logger.error('cooldown task is timeouted') - // throw new Exception("cooldown task is timeouted after 10 mins") - // } - // Thread.sleep(5000) - - // result = sql "show data FROM ${dbName}.${tableName}" - // sqlResult = result[0][5].toString(); - + // wait cooldown + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + int count = 0; + while (sqlResult.contains("0.00")) { + if (++count >= 120) { // 10min + logger.error('cooldown task is timeouted') + throw new Exception("cooldown task is timeouted after 10 mins") + } + Thread.sleep(5000) - // } + result = sql "show data FROM ${dbName}.${tableName}" + sqlResult = result[0][5].toString(); + } - // assertNotEquals('0.000 ', result[0][5].toString()) + assertNotEquals('0.000 ', result[0][5].toString()) sql """ BACKUP SNAPSHOT ${dbName}.${snapshotName} @@ -374,11 +372,11 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { assertTrue(snapshot != null) // 1 老表存在的情况 - // 1.1 restore 不指定, 预期成功，且落冷 - // 1.2 restore 指定 （"reserve_storage_policy"="true"）， 预期成功，且落冷 - // 1.3 restore 指定 （"reserve_storage_policy"="false"），预期成功，且不落冷 + // 1.1 restore 不指定。预期失败, 不支持将冷热属性的表恢复到已存在的表中。 + // 1.2 restore 指定 （"reserve_storage_policy"="true"), 预期失败, 不支持将冷热属性的表恢复到已存在的表中。 + // 1.3 restore 指定 （"reserve_storage_policy"="false"), 预期成功，且不落冷 - // 2 删除表 + // 2 删除老表 // 1.1 restore 不指定 预期成功，且落冷 // 1.2 restore 指定 （"reserve_storage_policy"="true"）预期成功，且落冷 // 1.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 @@ -391,9 +389,9 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { // 1. old table exist - // 1.1 restore normal - // 1.2 restore with（"reserve_storage_policy"="true"） - // 1.3 restore with（"reserve_storage_policy"="false"） + // 1.1 restore normal fail + // 1.2 restore with（"reserve_storage_policy"="true") fail + // 1.3 restore with（"reserve_storage_policy"="false") success and don't cooldown logger.info(" ====================================== 1.1 ==================================== ") sql """ RESTORE SNAPSHOT ${dbName}.${snapshotName} @@ -466,9 +464,9 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { // 2. drop old table - // 2.1 restore normal - // 2.2 restore with （"reserve_storage_policy"="true"） - // 2.3 restore with （"reserve_storage_policy"="false"） + // 2.1 restore normal success and cooldown + // 2.2 restore with （"reserve_storage_policy"="true"）success and cooldown + // 2.3 restore with （"reserve_storage_policy"="false"）success and don't cooldown logger.info(" ====================================== 2.1 ==================================== ") sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" sql """ @@ -572,9 +570,9 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { // 3. drop old table and resource and policy - // 3.1 restore normal - // 3.2 restore with（"reserve_storage_policy"="true"） - // 3.3 restore with（"reserve_storage_policy"="false"） + // 3.1 restore normal success and cooldown + // 3.2 restore with（"reserve_storage_policy"="true"）success and cooldown + // 3.3 restore with（"reserve_storage_policy"="false"）success and don't cooldown logger.info(" ====================================== 3.1 ==================================== ") sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" try_sql """ @@ -717,7 +715,7 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { assertEquals(found, 0) - // 4. alter policy + // 4. alter policy and success logger.info(" ====================================== 4.1 ==================================== ") sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" sql """ @@ -945,35 +943,37 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { assertTrue(snapshot != null) // 1 老表存在的情况 - // 1.1 restore 指定 （"storage_resource"="resource_name1"）， 预期失败，路径需不一致 - // 1.2 restore 指定 （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 - // 1.3 restore 指定 （"storage_resource"="resource_new_name"）， 预期失败，有stroage policy属性的表不允许restore到已存在的表 + // 1.1 restore 指定 （"storage_resource"="resource_name_exist"), 预期失败，不支持将冷热属性的表恢复到已存在的表中。 + // 1.2 restore 指定 （"storage_resource"="resource_name_not_exist"), 预期失败，resource不存在 + // 1.3 restore 指定 （"storage_resource"="resource_new_name"), 预期失败，不支持将冷热属性的表恢复到已存在的表中。 // 2 删除表 - // 2.1 restore 指定 （"storage_resource"="resource_name1"）， 预期失败，resource路径需不一致 - // 2.2 restore 指定 （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 - // 2.3 restore 指定 （"storage_resource"="resource_new_name"）， storage policy 存在失败 + // 2.1 restore 指定 （"storage_resource"="resource_name_exist"), 预期失败，resource路径需不一致 + // 2.2 restore 指定 （"storage_resource"="resource_name_not_exist"), 预期失败，resource不存在 + // 2.3 restore 指定 （"storage_resource"="resource_new_name"), storage policy 存在失败 // 3 删除表和policy - // 3.1 restore 指定 （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 - // 3.2 restore 指定 （"storage_resource"="resource_new_name"）， 成功 + // 3.1 restore 指定 （"storage_resource"="resource_name_not_exist"), 预期失败，resource不存在 + // 3.2 restore 指定 （"storage_resource"="resource_new_name"), 成功 // 4 删除表和policy 同时指定storage_resource和reserve_storage_policy - // 4.1 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="true"），预期失败，resource不存在 - // 4.2 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="false"），预期失败，resource不存在 - // 4.3 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="true"），预期成功 - // 4.4 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="false"），预期成功 + // 4.1 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="true"), 预期失败，resource不存在 + // 4.2 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="false"), 预期失败，resource不存在 + // 4.3 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="true"), 预期成功,且落冷 + // 4.4 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="false"), 预期成功，且不落冷 + + // 1 old table exist - // 1.1 restore with （"storage_resource"="resource_name1"）， 预期失败，路径需不一致 - // 1.2 restore with （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 - // 1.3 restore with （"storage_resource"="resource_new_name"）， 预期失败，有stroage policy属性的表不允许restore到已存在的表 + // 1.1 restore with （"storage_resource"="resource_name1") fail + // 1.2 restore with （"storage_resource"="resource_name_not_exist") fail + // 1.3 restore with （"storage_resource"="resource_new_name") fail logger.info(" ====================================== 1.1 ==================================== ") sql """ RESTORE SNAPSHOT ${dbName}.${snapshotName} @@ -1017,7 +1017,6 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { assertEquals(fail_restore_1, null) - // 1.3 restore 指定 （"storage_resource"="resource_new_name"）， 预期失败，有stroage policy属性的表不允许restore到已存在的表 logger.info(" ====================================== 1.3 ==================================== ") sql """ RESTORE SNAPSHOT ${dbName}.${snapshotName} @@ -1044,10 +1043,10 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { - // 2 删除表 - // 2.1 restore 指定 （"storage_resource"="resource_name1"）， 预期失败，resource路径需不一致 - // 2.2 restore 指定 （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 - // 2.3 restore 指定 （"storage_resource"="resource_new_name"）， storage policy 存在失败 + // 2 drop old table + // 2.1 restore with （"storage_resource"="resource_name_exist"）fail + // 2.2 restore with （"storage_resource"="resource_name_not_exist"） fail + // 2.3 restore with （"storage_resource"="resource_new_name"）fail logger.info(" ====================================== 2.1 ==================================== ") sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" sql """ @@ -1089,7 +1088,6 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { assertEquals(fail_restore_2, null) - // 2.3 restore 指定 （"storage_resource"="resource_new_name"）， storage policy 存在失败 logger.info(" ====================================== 2.3 ==================================== ") sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" sql """ @@ -1111,10 +1109,9 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { row = records[records.size() - 1] assertTrue(row.Status.contains("already exist but with different properties")) - - // 3 删除表和policy - // 3.1 restore 指定 （"storage_resource"="resource_name_not_exist"），预期失败，resource不存在 - // 3.2 restore 指定 （"storage_resource"="resource_new_name"）， 成功 + // 3 drop table and resource and policy + // 3.1 restore with （"storage_resource"="resource_name_not_exist") fail + // 3.2 restore with （"storage_resource"="resource_new_name") success and cooldown logger.info(" ====================================== 3.1 ==================================== ") sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" try_sql """ @@ -1203,11 +1200,11 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { - // 4 删除表和policy - // 4.1 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="true"），预期失败，resource不存在 - // 4.2 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="false"），预期失败，resource不存在 - // 4.3 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="true"），预期成功 - // 4.4 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="false"），预期成功 + // 4 drop table/resource/policy, set both storage_resource and reserve_storage_policy + // 4.1 restore with （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="true") fail + // 4.2 restore with （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="false") fail + // 4.3 restore with （"storage_resource"="resource_new_name", "reserve_storage_policy"="true") success and cooldown + // 4.4 restore with （"storage_resource"="resource_new_name", "reserve_storage_policy"="false") success and don't cooldown logger.info(" ====================================== 4.1 ==================================== ") sql "DROP TABLE if exists ${dbName}.${tableName}" try_sql """ @@ -1232,7 +1229,6 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { assertEquals(fail_restore_4, null) - // 4.2 restore 指定 （"storage_resource"="resource_name_not_exist", "reserve_storage_policy"="false"），预期成功 logger.info(" ====================================== 4.2 ==================================== ") sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" try_sql """ @@ -1258,7 +1254,6 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { assertEquals(fail_restore_5, null) - // 4.3 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="true"），预期成功 logger.info(" ====================================== 4.3 ==================================== ") sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" try_sql """ @@ -1326,7 +1321,6 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { - // 4.4 restore 指定 （"storage_resource"="resource_new_name", "reserve_storage_policy"="false"），预期成功 logger.info(" ====================================== 4.4 ==================================== ") sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" try_sql """ From 74869af47ffa9456bf282e529f1121ce4ffa75b2 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Mon, 18 Nov 2024 16:05:52 +0800 Subject: [PATCH 07/15] use new storage policy id --- .../org/apache/doris/backup/RestoreJob.java | 5 ++--- .../org/apache/doris/policy/PolicyMgr.java | 18 ++++++++++++++++++ .../test_backup_restore_cold_data.groovy | 12 ++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 441fc1c7a3cd9c..5ba0946ef4eab3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1419,10 +1419,9 @@ private void checkAndRestoreStoragePolicies() { } else { // restore storage policy try { - policyMgr.replayCreate(backupStoargePolicy); - Env.getCurrentEnv().getEditLog().logCreatePolicy(backupStoargePolicy); + policyMgr.createStoragePolicy(backupStoargePolicy); } catch (Exception e) { - LOG.error("restore user property fail should not happen", e); + LOG.error("restore storage policy fail should not happen", e); } storagePolicies.add(backupStoargePolicy); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java index f1c8dd27f8d3b1..772ffc5adab71d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/PolicyMgr.java @@ -146,6 +146,24 @@ public void createPolicy(CreatePolicyStmt stmt) throws UserException { } } + /** + * Create policy through StoragePolicy. + **/ + public void createStoragePolicy(StoragePolicy storagePolicy) throws UserException { + Map pros = Maps.newConcurrentMap(); + if (storagePolicy.getCooldownTimestampMs() != -1) { + pros.put(StoragePolicy.COOLDOWN_DATETIME, String.valueOf(storagePolicy.getCooldownTimestampMs())); + } + if (storagePolicy.getCooldownTtl() != -1) { + pros.put(StoragePolicy.COOLDOWN_TTL, String.valueOf(storagePolicy.getCooldownTtl())); + } + pros.put(StoragePolicy.STORAGE_RESOURCE, storagePolicy.getStorageResource()); + + CreatePolicyStmt stmt = new CreatePolicyStmt(storagePolicy.getType(), true, + storagePolicy.getPolicyName(), pros); + createPolicy(stmt); + } + /** * Create policy through http api. **/ diff --git a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy index 11d890a20ef019..8c41e2126768a5 100644 --- a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy +++ b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy @@ -377,15 +377,15 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { // 1.3 restore 指定 （"reserve_storage_policy"="false"), 预期成功，且不落冷 // 2 删除老表 - // 1.1 restore 不指定 预期成功，且落冷 - // 1.2 restore 指定 （"reserve_storage_policy"="true"）预期成功，且落冷 - // 1.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 + // 2.1 restore 不指定 预期成功，且落冷 + // 2.2 restore 指定 （"reserve_storage_policy"="true"）预期成功，且落冷 + // 2.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 // 3 删除resource 和 policy - // 1.1 restore 不指定 预期成功，且落冷 - // 1.2 restore 指定 （"reserve_storage_policy"="true"）预期成功，且落冷 - // 1.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 + // 2.1 restore 不指定 预期成功，且落冷 + // 2.2 restore 指定 （"reserve_storage_policy"="true"）预期成功，且落冷 + // 2.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 // 1. old table exist From e2e5dd3b609b6a42ac7a4b3f24eba56d0bad4fd8 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Mon, 18 Nov 2024 16:07:13 +0800 Subject: [PATCH 08/15] update FeMetaVersion.VERSION_130 --- .../java/org/apache/doris/common/FeMetaVersion.java | 5 +++-- .../main/java/org/apache/doris/backup/BackupMeta.java | 11 +++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index a81a01227e75f8..dc16162bca45fd 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -103,9 +103,10 @@ public final class FeMetaVersion { public static final int VERSION_140 = 140; + // For BackupMeta storage policy + public static final int VERSION_141 = 141; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_140; - + public static final int VERSION_CURRENT = VERSION_141; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java index 294b4fb3ff9495..57b441cf917809 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java @@ -184,10 +184,13 @@ public void readFields(DataInput in) throws IOException { Resource resource = Resource.read(in); resourceNameMap.put(resource.getName(), resource); } - size = in.readInt(); - for (int i = 0; i < size; i++) { - StoragePolicy policy = StoragePolicy.read(in); - storagePolicyNameMap.put(policy.getName(), policy); + + if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_141) { + size = in.readInt(); + for (int i = 0; i < size; i++) { + StoragePolicy policy = StoragePolicy.read(in); + storagePolicyNameMap.put(policy.getName(), policy); + } } } From 684c6cff1994c1c1575007187a76255a2fea380d Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Thu, 28 Nov 2024 11:17:28 +0800 Subject: [PATCH 09/15] update for review --- be/src/olap/olap_define.h | 2 +- be/src/olap/rowset/beta_rowset.cpp | 85 -------- be/src/olap/rowset/beta_rowset.h | 2 - be/src/olap/rowset/rowset.h | 2 - be/src/olap/snapshot_manager.cpp | 10 +- be/src/olap/tablet.cpp | 11 - be/src/olap/tablet.h | 2 - be/src/runtime/snapshot_loader.cpp | 23 ++- .../org/apache/doris/backup/BackupJob.java | 9 + .../org/apache/doris/backup/BackupMeta.java | 7 +- .../org/apache/doris/backup/RestoreJob.java | 11 +- .../apache/doris/catalog/PartitionInfo.java | 4 +- .../doris/service/FrontendServiceImpl.java | 1 + .../test_backup_restore_cold_data.groovy | 188 +++++++++++++++--- 14 files changed, 194 insertions(+), 163 deletions(-) diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h index bb767f9a9f6e69..24ce8572b98be1 100644 --- a/be/src/olap/olap_define.h +++ b/be/src/olap/olap_define.h @@ -99,7 +99,7 @@ static const std::string INCREMENTAL_DELTA_PREFIX = "incremental_delta"; static const std::string CLONE_PREFIX = "clone"; static const std::string SPILL_DIR_PREFIX = "spill"; static const std::string SPILL_GC_DIR_PREFIX = "spill_gc"; -static const std::string REMOTE_FILE_INFO = "remote_file_info"; +static const std::string REMOTE_SNAPSHOT_INFO = "remote_snapshot_info"; static inline std::string local_segment_path(std::string_view tablet_path, std::string_view rowset_id, int64_t seg_id) { diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 7d0ddd94c42233..8ca04925bfdd45 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -432,91 +432,6 @@ Status BetaRowset::copy_files_to(const std::string& dir, const RowsetId& new_row return Status::OK(); } -Status BetaRowset::download(const StorageResource& dest_fs, const std::string& dir) { - if (is_local()) { - DCHECK(false) << _rowset_meta->tablet_id() << ' ' << rowset_id(); - return Status::InternalError("should be remote rowset. tablet_id={} rowset_id={}", - _rowset_meta->tablet_id(), rowset_id().to_string()); - } - - if (num_segments() < 1) { - return Status::OK(); - } - - Status status; - std::vector linked_success_files; - Defer remove_linked_files {[&]() { // clear download files if errors happen - if (!status.ok()) { - LOG(WARNING) << "will delete download success files due to error " << status; - std::vector paths; - for (auto& file : linked_success_files) { - paths.emplace_back(file); - LOG(WARNING) << "will delete download success file " << file << " due to error"; - } - static_cast(dest_fs.fs->batch_delete(paths)); - LOG(WARNING) << "done delete download success files due to error " << status; - } - }}; - - for (int i = 0; i < num_segments(); ++i) { - // Note: Here we use relative path for remote. - auto remote_seg_path = - dest_fs.remote_segment_path(_rowset_meta->tablet_id(), rowset_id().to_string(), i); - - auto local_seg_path = local_segment_path(dir, rowset_id().to_string(), i); - - RETURN_IF_ERROR(dest_fs.fs->download(remote_seg_path, local_seg_path)); - - linked_success_files.push_back(local_seg_path); - - if (_schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1) { - if (_schema->has_inverted_index()) { - std::string inverted_index_src_file = - InvertedIndexDescriptor::get_index_file_path_v2( - InvertedIndexDescriptor::get_index_file_path_prefix( - remote_seg_path)); - - std::string inverted_index_dst_file_path = - InvertedIndexDescriptor::get_index_file_path_v2( - InvertedIndexDescriptor::get_index_file_path_prefix( - local_seg_path)); - - RETURN_IF_ERROR(dest_fs.fs->download(inverted_index_src_file, - inverted_index_dst_file_path)); - - linked_success_files.push_back(inverted_index_dst_file_path); - } - } else { - for (const auto& index : _schema->inverted_indexes()) { - if (index->index_type() != IndexType::INVERTED) { - continue; - } - - auto index_id = index->index_id(); - std::string inverted_index_src_file = - InvertedIndexDescriptor::get_index_file_path_v1( - InvertedIndexDescriptor::get_index_file_path_prefix( - remote_seg_path), - index_id, index->get_index_suffix()); - - std::string inverted_index_dst_file_path = - InvertedIndexDescriptor::get_index_file_path_v1( - InvertedIndexDescriptor::get_index_file_path_prefix(local_seg_path), - index_id, index->get_index_suffix()); - - RETURN_IF_ERROR(dest_fs.fs->download(inverted_index_src_file, - inverted_index_dst_file_path)); - - linked_success_files.push_back(inverted_index_dst_file_path); - LOG(INFO) << "success to download. from=" << inverted_index_src_file << ", " - << "to=" << inverted_index_dst_file_path; - } - } - } - - return Status::OK(); -} - Status BetaRowset::upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) { if (!is_local()) { DCHECK(false) << _rowset_meta->tablet_id() << ' ' << rowset_id(); diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 4d36bb251de79e..52d5ac5c8a8742 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -62,8 +62,6 @@ class BetaRowset final : public Rowset { Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) override; - Status download(const StorageResource& dest_fs, const std::string& dir) override; - Status upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) override; // only applicable to alpha rowset, no op here diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index 4545c7ac6a1508..98d88ba19f2068 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -213,8 +213,6 @@ class Rowset : public std::enable_shared_from_this, public MetadataAdder // copy all files to `dir` virtual Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) = 0; - virtual Status download(const StorageResource& dest_fs, const std::string& dir) = 0; - virtual Status upload_to(const StorageResource& dest_fs, const RowsetId& new_rowset_id) = 0; virtual Status remove_old_files(std::vector* files_to_remove) = 0; diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index 4720415a553819..bf322f4e5cfdbc 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -171,7 +171,9 @@ Result> SnapshotManager::convert_rowset_ids( new_tablet_meta_pb.set_tablet_id(tablet_id); *new_tablet_meta_pb.mutable_tablet_uid() = TabletUid::gen_uid().to_proto(); new_tablet_meta_pb.set_replica_id(replica_id); - new_tablet_meta_pb.set_storage_policy_id(storage_policy_id); + if (storage_policy_id > 0) { + new_tablet_meta_pb.set_storage_policy_id(storage_policy_id); + } if (table_id > 0) { new_tablet_meta_pb.set_table_id(table_id); } @@ -602,9 +604,9 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet std::string delimeter = "|"; if (!have_remote_file) { - auto romote_file_info = - fmt::format("{}/{}", schema_full_path, REMOTE_FILE_INFO); - RETURN_IF_ERROR(io::global_local_filesystem()->create_file(romote_file_info, + auto romote_snapshot_info = + fmt::format("{}/{}", schema_full_path, REMOTE_SNAPSHOT_INFO); + RETURN_IF_ERROR(io::global_local_filesystem()->create_file(romote_snapshot_info, &file_writer)); RETURN_IF_ERROR(file_writer->append( std::to_string(rs->rowset_meta()->tablet_id()))); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 9f911be33f1ee9..a1a56507ffc67a 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2025,17 +2025,6 @@ Status Tablet::cooldown(RowsetSharedPtr rowset) { return Status::OK(); } -Status Tablet::download(RowsetSharedPtr rowset, const std::string& dir) { - Status st; - auto storage_resource = DORIS_TRY(get_resource_by_storage_policy_id(storage_policy_id())); - - if (st = rowset->download(storage_resource, dir); !st.ok()) { - return st; - } - - return Status::OK(); -} - // hold SHARED `cooldown_conf_lock` Status Tablet::_cooldown_data(RowsetSharedPtr rowset) { DCHECK(_cooldown_conf.cooldown_replica_id == replica_id()); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index f280688a7f63ff..40b911d6391b9b 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -378,8 +378,6 @@ class Tablet final : public BaseTablet { // Cooldown to remote fs. Status cooldown(RowsetSharedPtr rowset = nullptr); - Status download(RowsetSharedPtr rowset, const std::string& dir); - RowsetSharedPtr pick_cooldown_rowset(); RowsetSharedPtr need_cooldown(int64_t* cooldown_timestamp, size_t* file_size); diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index bfe8618d1e976c..9fca72183cd069 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -46,6 +46,7 @@ #include "io/fs/remote_file_system.h" #include "io/fs/s3_file_system.h" #include "io/hdfs_builder.h" +#include "olap/olap_define.h" #include "olap/data_dir.h" #include "olap/snapshot_manager.h" #include "olap/storage_engine.h" @@ -163,16 +164,16 @@ static Status download_and_upload_one_file(io::RemoteFileSystem& dest_fs, static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet_id, const std::string& local_path, const std::string& dest_path, - io::RemoteFileSystem* cold_fs, const std::string& rowset, + io::RemoteFileSystem* cold_fs, const std::string& rowset_id, int segments, int have_inverted_index) { Status res = Status::OK(); std::string remote_tablet_path = fmt::format("{}/{}", DATA_PREFIX, tablet_id); for (int i = 0; i < segments; i++) { - std::string remote_seg_path = fmt::format("{}/{}_{}.dat", remote_tablet_path, rowset, i); - std::string local_seg_path = fmt::format("{}/{}_{}.dat", local_path, rowset, i); - std::string dest_seg_path = fmt::format("{}/{}_{}.dat", dest_path, rowset, i); + std::string remote_seg_path = fmt::format("{}/{}_{}.dat", remote_tablet_path, rowset_id, i); + std::string local_seg_path = fmt::format("{}/{}_{}.dat", local_path, rowset_id, i); + std::string dest_seg_path = fmt::format("{}/{}_{}.dat", dest_path, rowset_id, i); RETURN_IF_ERROR(download_and_upload_one_file(dest_fs, cold_fs, remote_seg_path, local_seg_path, dest_seg_path)); @@ -183,7 +184,7 @@ static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet } std::vector remote_index_files; - RETURN_IF_ERROR(list_segment_inverted_index_file(cold_fs, remote_tablet_path, rowset, + RETURN_IF_ERROR(list_segment_inverted_index_file(cold_fs, remote_tablet_path, rowset_id, &remote_index_files)); for (auto& index_file : remote_index_files) { @@ -199,12 +200,12 @@ static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_id, const std::string& local_path, const std::string& dest_path, - const std::string& remote_file) { + const std::string& remote_snapshot_info) { io::FileReaderSPtr file_reader; Status res = Status::OK(); - std::string full_remote_path = local_path + '/' + remote_file; - RETURN_IF_ERROR(io::global_local_filesystem()->open_file(full_remote_path, &file_reader)); + std::string full_remote_snapshot_info_path = local_path + '/' + remote_snapshot_info; + RETURN_IF_ERROR(io::global_local_filesystem()->open_file(full_remote_snapshot_info_path, &file_reader)); size_t bytes_read = 0; char* buff = (char*)malloc(file_reader->size() + 1); RETURN_IF_ERROR(file_reader->read_at(0, {buff, file_reader->size()}, &bytes_read)); @@ -300,7 +301,7 @@ Status SnapshotLoader::upload(const std::map& src_to_d for (auto& local_file : local_files) { RETURN_IF_ERROR(_report_every(10, &report_counter, finished_num, total_num, TTaskType::type::UPLOAD)); - if (local_file.compare("remote_file_info") == 0) { + if (local_file.compare(REMOTE_SNAPSHOT_INFO) == 0) { RETURN_IF_ERROR(upload_remote_file(*_remote_fs, tablet_id, src_path, dest_path, local_file)); } @@ -421,7 +422,7 @@ Status SnapshotLoader::download(const std::map& src_to const FileStat& file_stat = iter.second; auto find = std::find(local_files.begin(), local_files.end(), remote_file); if (find == local_files.end()) { - if (remote_file.compare(REMOTE_FILE_INFO) == 0) { + if (remote_file.compare(REMOTE_SNAPSHOT_INFO) == 0) { continue; } // remote file does not exist in local, download it @@ -430,7 +431,7 @@ Status SnapshotLoader::download(const std::map& src_to if (_end_with(remote_file, ".hdr")) { // this is a header file, download it. need_download = true; - } else if (remote_file.compare(REMOTE_FILE_INFO) == 0) { + } else if (remote_file.compare(REMOTE_SNAPSHOT_INFO) == 0) { continue; } else { // check checksum diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index f2c910341c931b..32c49e7d01f9a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -519,6 +519,15 @@ private void prepareAndSendSnapshotTask() { } prepareBackupMetaForOlapTableWithoutLock(tableRef, olapTable, copiedTables, copiedStoragePolicys); + for (StoragePolicy policy : copiedStoragePolicys) { + Resource resource = Env.getCurrentEnv().getResourceMgr() + .getResource(policy.getStorageResource()); + if (resource.getType() != Resource.ResourceType.S3) { + status = new Status(ErrCode.COMMON_ERROR, + "backup job only support S3 type storage policy:" + resource.getType()); + return; + } + } break; case VIEW: prepareBackupMetaForViewWithoutLock((View) tbl, copiedTables); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java index 57b441cf917809..82323d97b6abe8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java @@ -60,7 +60,7 @@ public class BackupMeta implements Writable, GsonPostProcessable { private BackupMeta() { } - public BackupMeta(List
tables, List resources, List storagePolicys) { + public BackupMeta(List
tables, List resources, List storagePolicies) { for (Table table : tables) { tblNameMap.put(table.getName(), table); tblIdMap.put(table.getId(), table); @@ -69,7 +69,7 @@ public BackupMeta(List
tables, List resources, List tables, List resources, List entry : partitionIdMap.entrySet()) { - idToDataProperty.put(entry.getKey(), reserveStoragePolicy - ? origIdToDataProperty.get(entry.getValue()) : DataProperty.DEFAULT_HDD_DATA_PROPERTY); + idToDataProperty.put(entry.getKey(), reserveStoragePolicy ? origIdToDataProperty.get(entry.getValue()) : + new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM)); idToReplicaAllocation.put(entry.getKey(), restoreReplicaAlloc == null ? origIdToReplicaAllocation.get(entry.getValue()) : restoreReplicaAlloc); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 3d7ed79524b738..cd47dc6b46e17f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -351,6 +351,7 @@ public TConfirmUnusedRemoteFilesResult confirmUnusedRemoteFiles(TConfirmUnusedRe List runningBackupJobs = jobs.stream().filter(job -> job instanceof BackupJob) .filter(job -> !((BackupJob) job).isDone()) + .filter(job -> ((BackupJob) job).getBackupMeta().getTable((info.tablet_id)) != null) .map(job -> (BackupJob) job).collect(Collectors.toList()); if (runningBackupJobs.size() > 0) { diff --git a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy index 8c41e2126768a5..b99e3e86b47f4d 100644 --- a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy +++ b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy @@ -170,18 +170,10 @@ suite("test_backup_cooldown", "backup_cooldown_data") { drop storage policy ${policy_name1}; """ - sql """ - drop resource ${resource_name1}; - """ - sql """ drop storage policy ${policy_name2}; """ - sql """ - drop resource ${resource_name2}; - """ - sql """ RESTORE SNAPSHOT ${dbName}.${snapshotName} FROM `${repoName}` @@ -382,14 +374,19 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { // 2.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 - // 3 删除resource 和 policy - // 2.1 restore 不指定 预期成功，且落冷 - // 2.2 restore 指定 （"reserve_storage_policy"="true"）预期成功，且落冷 - // 2.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 + // 3 删除老表和policy + // 3.1 restore 不指定 预期成功，且落冷 + // 3.2 restore 指定 （"reserve_storage_policy"="true"）预期成功，且落冷 + // 3.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 + + // 4 删除老表和resource、policy + // 4.1 restore 不指定 预期失败，resource不存在 + // 4.2 restore 指定 （"reserve_storage_policy"="true"）预期失败，resource不存在 + // 4.3 restore 指定 （"reserve_storage_policy"="false"）预期成功，且不落冷 // 1. old table exist - // 1.1 restore normal fail + // 1.1 restore normal fail // 1.2 restore with（"reserve_storage_policy"="true") fail // 1.3 restore with（"reserve_storage_policy"="false") success and don't cooldown logger.info(" ====================================== 1.1 ==================================== ") @@ -408,7 +405,7 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { // restore failed records = sql_return_maparray "SHOW restore FROM ${dbName}" - row = records[records.size() - 1] + row = records[records.size() - 1] assertTrue(row.Status.contains("Can't restore remote partition")) result = sql "SELECT * FROM ${dbName}.${tableName}" @@ -569,7 +566,7 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { assertEquals(found, 0) - // 3. drop old table and resource and policy + // 3. drop old table and policy // 3.1 restore normal success and cooldown // 3.2 restore with（"reserve_storage_policy"="true"）success and cooldown // 3.3 restore with（"reserve_storage_policy"="false"）success and don't cooldown @@ -578,9 +575,6 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { try_sql """ drop storage policy ${policy_name1}; """ - try_sql """ - drop resource ${resource_name1}; - """ sql """ RESTORE SNAPSHOT ${dbName}.${snapshotName} FROM `${repoName}` @@ -619,9 +613,6 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { try_sql """ drop storage policy ${policy_name1}; """ - try_sql """ - drop resource ${resource_name1}; - """ sql """ RESTORE SNAPSHOT ${dbName}.${snapshotName} FROM `${repoName}` @@ -661,6 +652,121 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { try_sql """ drop storage policy ${policy_name1}; """ + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "reserve_storage_policy"="false" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + // check table don't have storage_policy + records = sql_return_maparray "show storage policy using" + found = 0 + for (def res2 : records) { + if (res2.Database.equals(dbName) && res2.Table.equals(tableName)) { + found = 1 + break + } + } + assertEquals(found, 0) + + // check storage policy ${policy_name1} not exist + records = sql_return_maparray "show storage policy" + found = 0 + for (def res2 : records) { + if (res2.PolicyName.equals(policy_name1)) { + found = 1 + break + } + } + assertEquals(found, 0) + + // check resource ${resource_name1} not exist + records = sql_return_maparray "show storage policy" + found = 0 + for (def res2 : records) { + if (res2.Name.equals(resource_name1)) { + found = 1 + break + } + } + assertEquals(found, 0) + + + // 4. drop old table and resource and policy + // 4.1 restore normal fail + // 4.2 restore with（"reserve_storage_policy"="true"） fail + // 4.3 restore with（"reserve_storage_policy"="false"）success and don't cooldown + logger.info(" ====================================== 4.1 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop resource ${resource_name1}; + """ + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + // restore failed with local restore is not exist + records = sql_return_maparray "SHOW restore FROM ${dbName}" + row = records[records.size() - 1] + assertTrue(row.Status.contains("is not exist")) + + + logger.info(" ====================================== 4.2 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ + try_sql """ + drop resource ${resource_name1}; + """ + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true", + "reserve_storage_policy"="true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + // restore failed with local restore is not exist + records = sql_return_maparray "SHOW restore FROM ${dbName}" + row = records[records.size() - 1] + assertTrue(row.Status.contains("is not exist")) + + + logger.info(" ====================================== 4.3 ==================================== ") + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop storage policy ${policy_name1}; + """ try_sql """ drop resource ${resource_name1}; """ @@ -689,7 +795,7 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { found = 1 break } - } + } assertEquals(found, 0) // check storage policy ${policy_name1} not exist @@ -700,7 +806,7 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { found = 1 break } - } + } assertEquals(found, 0) // check resource ${resource_name1} not exist @@ -711,13 +817,31 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { found = 1 break } - } + } assertEquals(found, 0) - - // 4. alter policy and success - logger.info(" ====================================== 4.1 ==================================== ") + // 5. alter policy and success + logger.info(" ====================================== 5.1 ==================================== ") sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + try_sql """ + drop resource ${resource_name1}; + """ + sql """ + CREATE RESOURCE IF NOT EXISTS "${resource_name1}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "AWS_REGION" = "${getS3Region()}", + "AWS_ROOT_PATH" = "regression/cooldown1", + "AWS_ACCESS_KEY" = "${getS3AK()}", + "AWS_SECRET_KEY" = "${getS3SK()}", + "AWS_MAX_CONNECTIONS" = "50", + "AWS_REQUEST_TIMEOUT_MS" = "3000", + "AWS_CONNECTION_TIMEOUT_MS" = "1000", + "AWS_BUCKET" = "${getS3BucketName()}", + "s3_validity_check" = "true" + ); + """ sql """ ALTER STORAGE POLICY ${policy_name2} PROPERTIES ("cooldown_ttl" = "11"); """ @@ -762,7 +886,7 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { found = 1 break } - } + } assertEquals(found, 1) @@ -968,8 +1092,6 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { - - // 1 old table exist // 1.1 restore with （"storage_resource"="resource_name1") fail // 1.2 restore with （"storage_resource"="resource_name_not_exist") fail @@ -1184,7 +1306,7 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { found = 1 break } - } + } assertEquals(found, 1) // check plocy_name2 storage_resource change to resource_new_name @@ -1300,7 +1422,7 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { found = 1 break } - } + } assertEquals(found, 1) // wait cooldown @@ -1357,7 +1479,7 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { found = 1 break } - } + } assertEquals(found, 0) From 75ff4b9b3395bc95890d1669581e9499c73e5608 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Thu, 28 Nov 2024 19:09:36 +0800 Subject: [PATCH 10/15] remove remote_file_info --- be/src/olap/olap_define.h | 1 - be/src/olap/snapshot_manager.cpp | 60 ++------------- be/src/runtime/snapshot_loader.cpp | 115 +++++++++++++++++++---------- 3 files changed, 81 insertions(+), 95 deletions(-) diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h index 24ce8572b98be1..5131c51ca01c20 100644 --- a/be/src/olap/olap_define.h +++ b/be/src/olap/olap_define.h @@ -99,7 +99,6 @@ static const std::string INCREMENTAL_DELTA_PREFIX = "incremental_delta"; static const std::string CLONE_PREFIX = "clone"; static const std::string SPILL_DIR_PREFIX = "spill"; static const std::string SPILL_GC_DIR_PREFIX = "spill_gc"; -static const std::string REMOTE_SNAPSHOT_INFO = "remote_snapshot_info"; static inline std::string local_segment_path(std::string_view tablet_path, std::string_view rowset_id, int64_t seg_id) { diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index bf322f4e5cfdbc..2ab635befae32f 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -206,6 +206,8 @@ Result> SnapshotManager::convert_rowset_ids( } else { // remote rowset *rowset_meta = visible_rowset; + //todo + rowset_meta->clear_resource_id(); } rowset_meta->set_tablet_id(tablet_id); @@ -465,7 +467,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet "missed version is a cooldowned rowset, must make full " "snapshot. missed_version={}, tablet_id={}", missed_version, ref_tablet->tablet_id()); - //todozy break; } consistent_rowsets.push_back(rowset); @@ -571,10 +572,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet } std::vector rs_metas; - RowsetMetaSharedPtr rsm; - bool have_remote_file = false; - io::FileWriterPtr file_writer; - for (auto& rs : consistent_rowsets) { if (rs->is_local()) { // local rowset @@ -582,57 +579,13 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet if (!res.ok()) { break; } - rsm = rs->rowset_meta(); - } else { - std::string rowset_meta_str; - RowsetMetaPB rs_meta_pb; - rs->rowset_meta()->to_rowset_pb(&rs_meta_pb); - rs_meta_pb.SerializeToString(&rowset_meta_str); - - RowsetMetaSharedPtr rowset_meta(new RowsetMeta()); - rowset_meta->init(rowset_meta_str); - - rsm = rowset_meta; - - // save_remote_file info - // tableid|storage_policy_id| - // rowset_id|num_segments|has_inverted_index| - // ...... - // rowset_id|num_segments|has_inverted_index - { - // write file - std::string delimeter = "|"; - - if (!have_remote_file) { - auto romote_snapshot_info = - fmt::format("{}/{}", schema_full_path, REMOTE_SNAPSHOT_INFO); - RETURN_IF_ERROR(io::global_local_filesystem()->create_file(romote_snapshot_info, - &file_writer)); - RETURN_IF_ERROR(file_writer->append( - std::to_string(rs->rowset_meta()->tablet_id()))); - RETURN_IF_ERROR(file_writer->append(delimeter)); - RETURN_IF_ERROR(file_writer->append( - std::to_string(ref_tablet->tablet_meta()->storage_policy_id()))); - have_remote_file = true; - } - RETURN_IF_ERROR(file_writer->append(delimeter)); - RETURN_IF_ERROR(file_writer->append(rs->rowset_id().to_string())); - RETURN_IF_ERROR(file_writer->append(delimeter)); - RETURN_IF_ERROR(file_writer->append(std::to_string(rs->num_segments()))); - RETURN_IF_ERROR(file_writer->append(delimeter)); - RETURN_IF_ERROR(file_writer->append( - std::to_string(rs->tablet_schema()->has_inverted_index()))); - } } - rs_metas.push_back(rsm); + rs_metas.push_back(rs->rowset_meta()); VLOG_NOTICE << "add rowset meta to clone list. " - << " start version " << rsm->start_version() << " end version " - << rsm->end_version() << " empty " << rsm->empty(); + << " start version " << rs->rowset_meta()->start_version() << " end version " + << rs->rowset_meta()->end_version() << " empty " << rs->rowset_meta()->empty(); } - if (have_remote_file) { - RETURN_IF_ERROR(file_writer->close()); - } if (!res.ok()) { LOG(WARNING) << "fail to create hard link. path=" << snapshot_id_path << " tablet=" << target_tablet->tablet_id() @@ -649,9 +602,6 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet new_tablet_meta->revise_delete_bitmap_unlocked(delete_bitmap_snapshot); } - //clear cooldown meta - new_tablet_meta->revise_clear_resource_id(); - if (snapshot_version == g_Types_constants.TSNAPSHOT_REQ_VERSION2) { res = new_tablet_meta->save(header_path); if (res.ok() && request.__isset.is_copy_tablet_task && request.is_copy_tablet_task) { diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index 9fca72183cd069..e9aec1e1666157 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -200,50 +200,32 @@ static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_id, const std::string& local_path, const std::string& dest_path, - const std::string& remote_snapshot_info) { - io::FileReaderSPtr file_reader; + const std::string& hdr_file) { Status res = Status::OK(); - std::string full_remote_snapshot_info_path = local_path + '/' + remote_snapshot_info; - RETURN_IF_ERROR(io::global_local_filesystem()->open_file(full_remote_snapshot_info_path, &file_reader)); - size_t bytes_read = 0; - char* buff = (char*)malloc(file_reader->size() + 1); - RETURN_IF_ERROR(file_reader->read_at(0, {buff, file_reader->size()}, &bytes_read)); - string str(buff, file_reader->size()); - size_t start = 0; - string delimiter = "|"; - size_t end = str.find(delimiter); - int64_t tablet_id_tmp = std::stol(str.substr(start, end - start)); - start = end + delimiter.length(); - - if (tablet_id_tmp != tablet_id) { - return Status::InternalError("Invalid tablet {}", tablet_id_tmp); + auto tablet_meta = std::make_shared(); + res = tablet_meta->create_from_file(local_path + "/" + hdr_file); + if (!res.ok()) { + return Status::Error( + "fail to load tablet_meta. file_path={}", local_path + "/" + hdr_file); } - end = str.find(delimiter, start); // - int64_t storage_policy_id = std::stol(str.substr(start, end - start)); - start = end + delimiter.length(); + if (tablet_meta->tablet_id() != tablet_id) { + return Status::InternalError("Invalid tablet {}", tablet_meta->tablet_id()); + } string rowset_id; int segments; int have_inverted_index; - auto storage_resource = DORIS_TRY(get_resource_by_storage_policy_id(storage_policy_id)); - - while (end != std::string::npos) { - end = str.find(delimiter, start); // - rowset_id = str.substr(start, end - start); - start = end + delimiter.length(); + auto storage_resource = DORIS_TRY(get_resource_by_storage_policy_id(tablet_meta->storage_policy_id())); - end = str.find(delimiter, start); - segments = std::stoi(str.substr(start, end - start)); - start = end + delimiter.length(); + for (auto rowset_meta : tablet_meta->all_rs_metas()) { + rowset_id = rowset_meta->rowset_id().to_string(); + segments = rowset_meta->num_segments(); + have_inverted_index = rowset_meta->tablet_schema()->has_inverted_index(); - end = str.find(delimiter, start); - have_inverted_index = std::stoi(str.substr(start, end - start)); - start = end + delimiter.length(); - - if (segments > 0) { + if (segments > 0 && !rowset_meta->is_local()) { RETURN_IF_ERROR(upload_remote_rowset(dest_fs, tablet_id, local_path, dest_path, storage_resource.fs.get(), rowset_id, segments, have_inverted_index)); @@ -253,6 +235,62 @@ static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_i return res; } + +// static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_id, +// const std::string& local_path, const std::string& dest_path, +// const std::string& remote_snapshot_info) { +// io::FileReaderSPtr file_reader; +// Status res = Status::OK(); + +// std::string full_remote_snapshot_info_path = local_path + '/' + remote_snapshot_info; +// RETURN_IF_ERROR(io::global_local_filesystem()->open_file(full_remote_snapshot_info_path, &file_reader)); +// size_t bytes_read = 0; +// char* buff = (char*)malloc(file_reader->size() + 1); +// RETURN_IF_ERROR(file_reader->read_at(0, {buff, file_reader->size()}, &bytes_read)); +// string str(buff, file_reader->size()); +// size_t start = 0; +// string delimiter = "|"; +// size_t end = str.find(delimiter); +// int64_t tablet_id_tmp = std::stol(str.substr(start, end - start)); +// start = end + delimiter.length(); + +// if (tablet_id_tmp != tablet_id) { +// return Status::InternalError("Invalid tablet {}", tablet_id_tmp); +// } + +// end = str.find(delimiter, start); // +// int64_t storage_policy_id = std::stol(str.substr(start, end - start)); +// start = end + delimiter.length(); + +// string rowset_id; +// int segments; +// int have_inverted_index; + +// auto storage_resource = DORIS_TRY(get_resource_by_storage_policy_id(storage_policy_id)); + +// while (end != std::string::npos) { +// end = str.find(delimiter, start); // +// rowset_id = str.substr(start, end - start); +// start = end + delimiter.length(); + +// end = str.find(delimiter, start); +// segments = std::stoi(str.substr(start, end - start)); +// start = end + delimiter.length(); + +// end = str.find(delimiter, start); +// have_inverted_index = std::stoi(str.substr(start, end - start)); +// start = end + delimiter.length(); + +// if (segments > 0) { +// RETURN_IF_ERROR(upload_remote_rowset(dest_fs, tablet_id, local_path, dest_path, +// storage_resource.fs.get(), rowset_id, segments, +// have_inverted_index)); +// } +// } + +// return res; +// } + Status SnapshotLoader::upload(const std::map& src_to_dest_path, std::map>* tablet_files) { if (!_remote_fs) { @@ -301,7 +339,11 @@ Status SnapshotLoader::upload(const std::map& src_to_d for (auto& local_file : local_files) { RETURN_IF_ERROR(_report_every(10, &report_counter, finished_num, total_num, TTaskType::type::UPLOAD)); - if (local_file.compare(REMOTE_SNAPSHOT_INFO) == 0) { + //if (local_file.compare(REMOTE_SNAPSHOT_INFO) == 0) { + if (_end_with(local_file, ".hdr")) { + // auto cloned_tablet_meta = std::make_shared(); + // cloned_tablet_meta->create_from_file(src_path + "/" + local_file); + RETURN_IF_ERROR(upload_remote_file(*_remote_fs, tablet_id, src_path, dest_path, local_file)); } @@ -422,17 +464,12 @@ Status SnapshotLoader::download(const std::map& src_to const FileStat& file_stat = iter.second; auto find = std::find(local_files.begin(), local_files.end(), remote_file); if (find == local_files.end()) { - if (remote_file.compare(REMOTE_SNAPSHOT_INFO) == 0) { - continue; - } // remote file does not exist in local, download it need_download = true; } else { if (_end_with(remote_file, ".hdr")) { // this is a header file, download it. need_download = true; - } else if (remote_file.compare(REMOTE_SNAPSHOT_INFO) == 0) { - continue; } else { // check checksum std::string local_md5sum; From e2d619e4d70cc9357e905729d9373263851c0b92 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Thu, 28 Nov 2024 22:17:47 +0800 Subject: [PATCH 11/15] fix cooldown_meta_id --- be/src/io/fs/s3_file_system.h | 2 +- be/src/olap/single_replica_compaction.cpp | 2 +- be/src/olap/snapshot_manager.cpp | 23 +++--- be/src/olap/snapshot_manager.h | 2 +- be/src/olap/tablet_meta.cpp | 9 --- be/src/olap/tablet_meta.h | 2 - be/src/olap/task/engine_clone_task.cpp | 2 +- .../task/engine_storage_migration_task.cpp | 2 +- be/src/runtime/snapshot_loader.cpp | 77 ++++--------------- 9 files changed, 29 insertions(+), 92 deletions(-) diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h index 31d2e19a1a9a38..748338632ad3ff 100644 --- a/be/src/io/fs/s3_file_system.h +++ b/be/src/io/fs/s3_file_system.h @@ -120,7 +120,7 @@ class S3FileSystem final : public RemoteFileSystem { // so no need to concat with prefix abs_path = path; } else { - return std::filesystem::path(fmt::format("s3://{}/{}", _bucket, _prefix)) / path; + abs_path = std::filesystem::path(fmt::format("s3://{}/{}", _bucket, _prefix)) / path; } return Status::OK(); } diff --git a/be/src/olap/single_replica_compaction.cpp b/be/src/olap/single_replica_compaction.cpp index 0f8fc141790fd3..10cedf8c50ec26 100644 --- a/be/src/olap/single_replica_compaction.cpp +++ b/be/src/olap/single_replica_compaction.cpp @@ -321,7 +321,7 @@ Status SingleReplicaCompaction::_fetch_rowset(const TReplicaInfo& addr, const st RETURN_IF_ERROR(_download_files(tablet()->data_dir(), remote_url_prefix, local_path)); _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( local_path, _tablet->tablet_id(), tablet()->replica_id(), _tablet->table_id(), - _tablet->partition_id(), _tablet->schema_hash(), 0)); + _tablet->partition_id(), _tablet->schema_hash(), false, 0)); // 4: finish_clone: create output_rowset and link file return _finish_clone(local_data_path, rowset_version); } diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index 2ab635befae32f..5c230c44be183c 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -141,7 +141,7 @@ Status SnapshotManager::release_snapshot(const string& snapshot_path) { Result> SnapshotManager::convert_rowset_ids( const std::string& clone_dir, int64_t tablet_id, int64_t replica_id, int64_t table_id, - int64_t partition_id, int32_t schema_hash, int64_t storage_policy_id) { + int64_t partition_id, int32_t schema_hash, bool is_restore, int64_t storage_policy_id) { SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker); std::vector guards; // check clone dir existed @@ -171,8 +171,9 @@ Result> SnapshotManager::convert_rowset_ids( new_tablet_meta_pb.set_tablet_id(tablet_id); *new_tablet_meta_pb.mutable_tablet_uid() = TabletUid::gen_uid().to_proto(); new_tablet_meta_pb.set_replica_id(replica_id); - if (storage_policy_id > 0) { + if (is_restore) { new_tablet_meta_pb.set_storage_policy_id(storage_policy_id); + new_tablet_meta_pb.clear_cooldown_meta_id(); } if (table_id > 0) { new_tablet_meta_pb.set_table_id(table_id); @@ -206,8 +207,9 @@ Result> SnapshotManager::convert_rowset_ids( } else { // remote rowset *rowset_meta = visible_rowset; - //todo - rowset_meta->clear_resource_id(); + if (is_restore) { + rowset_meta->clear_resource_id(); + } } rowset_meta->set_tablet_id(tablet_id); @@ -495,11 +497,8 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet if (!is_single_rowset_clone && (!res.ok() || request.missing_version.empty())) { if (!request.__isset.missing_version && ref_tablet->tablet_meta()->cooldown_meta_id().initialized()) { - LOG(WARNING) << "currently not support backup tablet with cooldowned remote " - "data. tablet=" - << request.tablet_id; - // return Status::NotSupported( - // "currently not support backup tablet with cooldowned remote data"); + LOG(INFO) << "Backup tablet with cooldowned remote data. tablet=" + << request.tablet_id; } /// not all missing versions are found, fall back to full snapshot. res = Status::OK(); // reset res @@ -582,10 +581,10 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet } rs_metas.push_back(rs->rowset_meta()); VLOG_NOTICE << "add rowset meta to clone list. " - << " start version " << rs->rowset_meta()->start_version() << " end version " - << rs->rowset_meta()->end_version() << " empty " << rs->rowset_meta()->empty(); + << " start version " << rs->rowset_meta()->start_version() + << " end version " << rs->rowset_meta()->end_version() << " empty " + << rs->rowset_meta()->empty(); } - if (!res.ok()) { LOG(WARNING) << "fail to create hard link. path=" << snapshot_id_path << " tablet=" << target_tablet->tablet_id() diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h index 160b67f6befd1e..306a01b83513ba 100644 --- a/be/src/olap/snapshot_manager.h +++ b/be/src/olap/snapshot_manager.h @@ -53,7 +53,7 @@ class SnapshotManager { Result> convert_rowset_ids( const std::string& clone_dir, int64_t tablet_id, int64_t replica_id, int64_t table_id, - int64_t partition_id, int32_t schema_hash, int64_t storage_policy_id); + int64_t partition_id, int32_t schema_hash, bool is_restore, int64_t storage_policy_id); private: Status _calc_snapshot_id_path(const TabletSharedPtr& tablet, int64_t timeout_s, diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 5788205f7edac7..43b0d5d8bd0ae0 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -886,15 +886,6 @@ void TabletMeta::revise_rs_metas(std::vector&& rs_metas) { _stale_rs_metas.clear(); } -void TabletMeta::revise_clear_resource_id() { - for (auto rs : _rs_metas) { - rs->clear_resource_id(); - } - for (auto rs : _stale_rs_metas) { - rs->clear_resource_id(); - } -} - // This method should call after revise_rs_metas, since new rs_metas might be a subset // of original tablet, we should revise the delete_bitmap according to current rowset. // diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 303257ffb34036..25f6bcd569be43 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -198,8 +198,6 @@ class TabletMeta : public MetadataAdder { void modify_rs_metas(const std::vector& to_add, const std::vector& to_delete, bool same_version = false); - - void revise_clear_resource_id(); void revise_rs_metas(std::vector&& rs_metas); void revise_delete_bitmap_unlocked(const DeleteBitmap& delete_bitmap); diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index 64e9ca283ccc3e..5d6f9190742e22 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -459,7 +459,7 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir, // No need to try again with another BE _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( local_data_path, _clone_req.tablet_id, _clone_req.replica_id, _clone_req.table_id, - _clone_req.partition_id, _clone_req.schema_hash, 0)); + _clone_req.partition_id, _clone_req.schema_hash, false, 0)); break; } // clone copy from one backend return status; diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index cf9ac35b3f67a3..aaaf3e203280cf 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -161,7 +161,7 @@ Status EngineStorageMigrationTask::_gen_and_write_header_to_hdr_file( // rowset create time is useful when load tablet from meta to check which tablet is the tablet to load _pending_rs_guards = DORIS_TRY(_engine.snapshot_mgr()->convert_rowset_ids( full_path, tablet_id, _tablet->replica_id(), _tablet->table_id(), - _tablet->partition_id(), schema_hash, 0)); + _tablet->partition_id(), schema_hash, false, 0)); return Status::OK(); } diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index e9aec1e1666157..70cb67c648d599 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -46,8 +46,8 @@ #include "io/fs/remote_file_system.h" #include "io/fs/s3_file_system.h" #include "io/hdfs_builder.h" -#include "olap/olap_define.h" #include "olap/data_dir.h" +#include "olap/olap_define.h" #include "olap/snapshot_manager.h" #include "olap/storage_engine.h" #include "olap/storage_policy.h" @@ -198,6 +198,10 @@ static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet return res; } +/* + * get the cooldown data info from the hdr file, download the cooldown data and + * upload it to remote storage. + */ static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_id, const std::string& local_path, const std::string& dest_path, const std::string& hdr_file) { @@ -206,7 +210,7 @@ static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_i auto tablet_meta = std::make_shared(); res = tablet_meta->create_from_file(local_path + "/" + hdr_file); if (!res.ok()) { - return Status::Error( + return Status::Error( "fail to load tablet_meta. file_path={}", local_path + "/" + hdr_file); } @@ -214,11 +218,16 @@ static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_i return Status::InternalError("Invalid tablet {}", tablet_meta->tablet_id()); } + if (!tablet_meta->cooldown_meta_id().initialized()) { + return res; + } + string rowset_id; int segments; int have_inverted_index; - auto storage_resource = DORIS_TRY(get_resource_by_storage_policy_id(tablet_meta->storage_policy_id())); + auto storage_resource = + DORIS_TRY(get_resource_by_storage_policy_id(tablet_meta->storage_policy_id())); for (auto rowset_meta : tablet_meta->all_rs_metas()) { rowset_id = rowset_meta->rowset_id().to_string(); @@ -235,62 +244,6 @@ static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_i return res; } - -// static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_id, -// const std::string& local_path, const std::string& dest_path, -// const std::string& remote_snapshot_info) { -// io::FileReaderSPtr file_reader; -// Status res = Status::OK(); - -// std::string full_remote_snapshot_info_path = local_path + '/' + remote_snapshot_info; -// RETURN_IF_ERROR(io::global_local_filesystem()->open_file(full_remote_snapshot_info_path, &file_reader)); -// size_t bytes_read = 0; -// char* buff = (char*)malloc(file_reader->size() + 1); -// RETURN_IF_ERROR(file_reader->read_at(0, {buff, file_reader->size()}, &bytes_read)); -// string str(buff, file_reader->size()); -// size_t start = 0; -// string delimiter = "|"; -// size_t end = str.find(delimiter); -// int64_t tablet_id_tmp = std::stol(str.substr(start, end - start)); -// start = end + delimiter.length(); - -// if (tablet_id_tmp != tablet_id) { -// return Status::InternalError("Invalid tablet {}", tablet_id_tmp); -// } - -// end = str.find(delimiter, start); // -// int64_t storage_policy_id = std::stol(str.substr(start, end - start)); -// start = end + delimiter.length(); - -// string rowset_id; -// int segments; -// int have_inverted_index; - -// auto storage_resource = DORIS_TRY(get_resource_by_storage_policy_id(storage_policy_id)); - -// while (end != std::string::npos) { -// end = str.find(delimiter, start); // -// rowset_id = str.substr(start, end - start); -// start = end + delimiter.length(); - -// end = str.find(delimiter, start); -// segments = std::stoi(str.substr(start, end - start)); -// start = end + delimiter.length(); - -// end = str.find(delimiter, start); -// have_inverted_index = std::stoi(str.substr(start, end - start)); -// start = end + delimiter.length(); - -// if (segments > 0) { -// RETURN_IF_ERROR(upload_remote_rowset(dest_fs, tablet_id, local_path, dest_path, -// storage_resource.fs.get(), rowset_id, segments, -// have_inverted_index)); -// } -// } - -// return res; -// } - Status SnapshotLoader::upload(const std::map& src_to_dest_path, std::map>* tablet_files) { if (!_remote_fs) { @@ -339,11 +292,7 @@ Status SnapshotLoader::upload(const std::map& src_to_d for (auto& local_file : local_files) { RETURN_IF_ERROR(_report_every(10, &report_counter, finished_num, total_num, TTaskType::type::UPLOAD)); - //if (local_file.compare(REMOTE_SNAPSHOT_INFO) == 0) { if (_end_with(local_file, ".hdr")) { - // auto cloned_tablet_meta = std::make_shared(); - // cloned_tablet_meta->create_from_file(src_path + "/" + local_file); - RETURN_IF_ERROR(upload_remote_file(*_remote_fs, tablet_id, src_path, dest_path, local_file)); } @@ -934,7 +883,7 @@ Status SnapshotLoader::move(const std::string& snapshot_path, TabletSharedPtr ta // rename the rowset ids and tabletid info in rowset meta auto res = _engine.snapshot_mgr()->convert_rowset_ids( snapshot_path, tablet_id, tablet->replica_id(), tablet->table_id(), - tablet->partition_id(), schema_hash, tablet->storage_policy_id()); + tablet->partition_id(), schema_hash, true, tablet->storage_policy_id()); if (!res.has_value()) [[unlikely]] { auto err_msg = fmt::format("failed to convert rowsetids in snapshot: {}, tablet path: {}, err: {}", From 90b140746c19a32f20067a0b74cd832dec9fdd49 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Tue, 3 Dec 2024 12:35:46 +0800 Subject: [PATCH 12/15] fix review2 --- be/src/runtime/snapshot_loader.cpp | 134 +++++++++++------- .../doris/service/FrontendServiceImpl.java | 11 +- 2 files changed, 81 insertions(+), 64 deletions(-) diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index 70cb67c648d599..c608355469e6ca 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -143,16 +143,50 @@ static Status list_segment_inverted_index_file(io::RemoteFileSystem* cold_fs, return Status::OK(); } -static Status download_and_upload_one_file(io::RemoteFileSystem& dest_fs, - io::RemoteFileSystem* cold_fs, - const std::string& remote_seg_path, - const std::string& local_seg_path, - const std::string& dest_seg_path) { - RETURN_IF_ERROR(cold_fs->download(remote_seg_path, local_seg_path)); - +static Status check_need_upload(const std::string& src_path, const std::string& local_file, + std::map& remote_files, + std::vector* local_files_with_checksum, + std::string* md5sum, bool* need_upload) { // calc md5sum of localfile + RETURN_IF_ERROR(io::global_local_filesystem()->md5sum(src_path + "/" + local_file, md5sum)); + VLOG_CRITICAL << "get file checksum: " << local_file << ": " << *md5sum; + local_files_with_checksum->push_back(local_file + "." + *md5sum); + + // check if this local file need upload + auto find = remote_files.find(local_file); + if (find != remote_files.end()) { + if (*md5sum != find->second.md5) { + // remote storage file exist, but with different checksum + LOG(WARNING) << "remote file checksum is invalid. remote: " << find->first + << ", local: " << *md5sum; + // TODO(cmy): save these files and delete them later + *need_upload = true; + } + } else { + *need_upload = true; + } + + return Status::OK(); +} + +static Status download_and_upload_one_cold_file( + io::RemoteFileSystem& dest_fs, const StorageResource& cold_fs, + const std::string& remote_seg_path, const std::string& local_seg_path, + const std::string& dest_seg_path, const std::string& local_path, + const std::string& local_file, std::map& remote_files, + std::vector* local_files_with_checksum) { + RETURN_IF_ERROR(cold_fs.fs.get()->download(remote_seg_path, local_seg_path)); + + bool need_upload = false; std::string md5sum; - RETURN_IF_ERROR(io::global_local_filesystem()->md5sum(local_seg_path, &md5sum)); + RETURN_IF_ERROR(check_need_upload(local_path, local_file, remote_files, + local_files_with_checksum, &md5sum, &need_upload)); + + if (!need_upload) { + VLOG_CRITICAL << "cold file exist in remote path, no need to upload: " << local_file; + return Status::OK(); + ; + } RETURN_IF_ERROR(upload_with_checksum(dest_fs, local_seg_path, dest_seg_path, md5sum)); @@ -162,21 +196,26 @@ static Status download_and_upload_one_file(io::RemoteFileSystem& dest_fs, return Status::OK(); } -static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet_id, - const std::string& local_path, const std::string& dest_path, - io::RemoteFileSystem* cold_fs, const std::string& rowset_id, - int segments, int have_inverted_index) { +static Status upload_remote_cold_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet_id, + const std::string& local_path, const std::string& dest_path, + const StorageResource& cold_fs, + const std::string& rowset_id, int segments, + int have_inverted_index, + std::map& remote_files, + std::vector* local_files_with_checksum) { Status res = Status::OK(); std::string remote_tablet_path = fmt::format("{}/{}", DATA_PREFIX, tablet_id); for (int i = 0; i < segments; i++) { - std::string remote_seg_path = fmt::format("{}/{}_{}.dat", remote_tablet_path, rowset_id, i); - std::string local_seg_path = fmt::format("{}/{}_{}.dat", local_path, rowset_id, i); - std::string dest_seg_path = fmt::format("{}/{}_{}.dat", dest_path, rowset_id, i); - - RETURN_IF_ERROR(download_and_upload_one_file(dest_fs, cold_fs, remote_seg_path, - local_seg_path, dest_seg_path)); + std::string local_file = fmt::format("{}_{}.dat", rowset_id, i); + std::string remote_seg_path = cold_fs.remote_segment_path(tablet_id, rowset_id, i); + std::string local_seg_path = local_segment_path(local_path, rowset_id, i); + std::string dest_seg_path = local_segment_path(dest_path, rowset_id, i); + + RETURN_IF_ERROR(download_and_upload_one_cold_file( + dest_fs, cold_fs, remote_seg_path, local_seg_path, dest_seg_path, local_path, + local_file, remote_files, local_files_with_checksum)); } if (!have_inverted_index) { @@ -184,16 +223,17 @@ static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet } std::vector remote_index_files; - RETURN_IF_ERROR(list_segment_inverted_index_file(cold_fs, remote_tablet_path, rowset_id, - &remote_index_files)); + RETURN_IF_ERROR(list_segment_inverted_index_file(cold_fs.fs.get(), remote_tablet_path, + rowset_id, &remote_index_files)); for (auto& index_file : remote_index_files) { std::string remote_index_path = fmt::format("{}/{}", remote_tablet_path, index_file); std::string local_seg_path = fmt::format("{}/{}", local_path, index_file); std::string dest_seg_path = fmt::format("{}/{}", dest_path, index_file); - RETURN_IF_ERROR(download_and_upload_one_file(dest_fs, cold_fs, remote_index_path, - local_seg_path, dest_seg_path)); + RETURN_IF_ERROR(download_and_upload_one_cold_file( + dest_fs, cold_fs, remote_index_path, local_seg_path, dest_seg_path, local_path, + index_file, remote_files, local_files_with_checksum)); } return res; } @@ -202,16 +242,17 @@ static Status upload_remote_rowset(io::RemoteFileSystem& dest_fs, int64_t tablet * get the cooldown data info from the hdr file, download the cooldown data and * upload it to remote storage. */ -static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_id, - const std::string& local_path, const std::string& dest_path, - const std::string& hdr_file) { +static Status upload_remote_cold_file(io::RemoteFileSystem& dest_fs, int64_t tablet_id, + const std::string& local_path, const std::string& dest_path, + std::map& remote_files, + std::vector* local_files_with_checksum) { Status res = Status::OK(); - + std::string hdr_file = local_path + "/" + std::to_string(tablet_id) + ".hdr"; auto tablet_meta = std::make_shared(); - res = tablet_meta->create_from_file(local_path + "/" + hdr_file); + res = tablet_meta->create_from_file(hdr_file); if (!res.ok()) { return Status::Error( - "fail to load tablet_meta. file_path={}", local_path + "/" + hdr_file); + "fail to load tablet_meta. file_path={}", hdr_file); } if (tablet_meta->tablet_id() != tablet_id) { @@ -235,9 +276,9 @@ static Status upload_remote_file(io::RemoteFileSystem& dest_fs, int64_t tablet_i have_inverted_index = rowset_meta->tablet_schema()->has_inverted_index(); if (segments > 0 && !rowset_meta->is_local()) { - RETURN_IF_ERROR(upload_remote_rowset(dest_fs, tablet_id, local_path, dest_path, - storage_resource.fs.get(), rowset_id, segments, - have_inverted_index)); + RETURN_IF_ERROR(upload_remote_cold_rowset( + dest_fs, tablet_id, local_path, dest_path, storage_resource, rowset_id, + segments, have_inverted_index, remote_files, local_files_with_checksum)); } } @@ -292,31 +333,10 @@ Status SnapshotLoader::upload(const std::map& src_to_d for (auto& local_file : local_files) { RETURN_IF_ERROR(_report_every(10, &report_counter, finished_num, total_num, TTaskType::type::UPLOAD)); - if (_end_with(local_file, ".hdr")) { - RETURN_IF_ERROR(upload_remote_file(*_remote_fs, tablet_id, src_path, dest_path, - local_file)); - } - // calc md5sum of localfile - std::string md5sum; - RETURN_IF_ERROR( - io::global_local_filesystem()->md5sum(src_path + "/" + local_file, &md5sum)); - VLOG_CRITICAL << "get file checksum: " << local_file << ": " << md5sum; - local_files_with_checksum.push_back(local_file + "." + md5sum); - - // check if this local file need upload bool need_upload = false; - auto find = remote_files.find(local_file); - if (find != remote_files.end()) { - if (md5sum != find->second.md5) { - // remote storage file exist, but with different checksum - LOG(WARNING) << "remote file checksum is invalid. remote: " << find->first - << ", local: " << md5sum; - // TODO(cmy): save these files and delete them later - need_upload = true; - } - } else { - need_upload = true; - } + std::string md5sum; + RETURN_IF_ERROR(check_need_upload(src_path, local_file, remote_files, + &local_files_with_checksum, &md5sum, &need_upload)); if (!need_upload) { VLOG_CRITICAL << "file exist in remote path, no need to upload: " << local_file; @@ -329,6 +349,10 @@ Status SnapshotLoader::upload(const std::map& src_to_d RETURN_IF_ERROR(upload_with_checksum(*_remote_fs, local_path, remote_path, md5sum)); } // end for each tablet's local files + // 2.4. upload cooldown data files + RETURN_IF_ERROR(upload_remote_cold_file(*_remote_fs, tablet_id, src_path, dest_path, + remote_files, &local_files_with_checksum)); + tablet_files->emplace(tablet_id, local_files_with_checksum); finished_num++; LOG(INFO) << "finished to write tablet to remote. local path: " << src_path diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index cd47dc6b46e17f..95bd9de911d5ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -28,7 +28,6 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.backup.AbstractJob; import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Snapshot; import org.apache.doris.catalog.AutoIncrementGenerator; @@ -346,15 +345,9 @@ public TConfirmUnusedRemoteFilesResult confirmUnusedRemoteFiles(TConfirmUnusedRe return; } - List jobs = Env.getCurrentEnv().getBackupHandler() - .getJobs(tabletMeta.getDbId(), label -> true); + BackupJob backupJob = (BackupJob) Env.getCurrentEnv().getBackupHandler().getJob(tabletMeta.getDbId()); - List runningBackupJobs = jobs.stream().filter(job -> job instanceof BackupJob) - .filter(job -> !((BackupJob) job).isDone()) - .filter(job -> ((BackupJob) job).getBackupMeta().getTable((info.tablet_id)) != null) - .map(job -> (BackupJob) job).collect(Collectors.toList()); - - if (runningBackupJobs.size() > 0) { + if (!backupJob.isDone() && backupJob.getBackupMeta().getTable((tabletMeta.getTableId())) != null) { LOG.warn("Backup is running on this tablet {} ", info.tablet_id); return; } From f6b6d8584b8fbc56879b0048d785e907a73a05d9 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Wed, 4 Dec 2024 14:12:51 +0800 Subject: [PATCH 13/15] fix local_files_with_checksum --- be/src/runtime/snapshot_loader.cpp | 47 +++++++++++++----------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index c608355469e6ca..5f4d9fa9ddd51a 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -144,13 +144,11 @@ static Status list_segment_inverted_index_file(io::RemoteFileSystem* cold_fs, } static Status check_need_upload(const std::string& src_path, const std::string& local_file, - std::map& remote_files, - std::vector* local_files_with_checksum, - std::string* md5sum, bool* need_upload) { + std::map& remote_files, std::string* md5sum, + bool* need_upload) { // calc md5sum of localfile RETURN_IF_ERROR(io::global_local_filesystem()->md5sum(src_path + "/" + local_file, md5sum)); VLOG_CRITICAL << "get file checksum: " << local_file << ": " << *md5sum; - local_files_with_checksum->push_back(local_file + "." + *md5sum); // check if this local file need upload auto find = remote_files.find(local_file); @@ -173,19 +171,16 @@ static Status download_and_upload_one_cold_file( io::RemoteFileSystem& dest_fs, const StorageResource& cold_fs, const std::string& remote_seg_path, const std::string& local_seg_path, const std::string& dest_seg_path, const std::string& local_path, - const std::string& local_file, std::map& remote_files, - std::vector* local_files_with_checksum) { + const std::string& local_file, std::map& remote_files) { RETURN_IF_ERROR(cold_fs.fs.get()->download(remote_seg_path, local_seg_path)); bool need_upload = false; std::string md5sum; - RETURN_IF_ERROR(check_need_upload(local_path, local_file, remote_files, - local_files_with_checksum, &md5sum, &need_upload)); + RETURN_IF_ERROR(check_need_upload(local_path, local_file, remote_files, &md5sum, &need_upload)); if (!need_upload) { VLOG_CRITICAL << "cold file exist in remote path, no need to upload: " << local_file; return Status::OK(); - ; } RETURN_IF_ERROR(upload_with_checksum(dest_fs, local_seg_path, dest_seg_path, md5sum)); @@ -201,8 +196,7 @@ static Status upload_remote_cold_rowset(io::RemoteFileSystem& dest_fs, int64_t t const StorageResource& cold_fs, const std::string& rowset_id, int segments, int have_inverted_index, - std::map& remote_files, - std::vector* local_files_with_checksum) { + std::map& remote_files) { Status res = Status::OK(); std::string remote_tablet_path = fmt::format("{}/{}", DATA_PREFIX, tablet_id); @@ -213,9 +207,9 @@ static Status upload_remote_cold_rowset(io::RemoteFileSystem& dest_fs, int64_t t std::string local_seg_path = local_segment_path(local_path, rowset_id, i); std::string dest_seg_path = local_segment_path(dest_path, rowset_id, i); - RETURN_IF_ERROR(download_and_upload_one_cold_file( - dest_fs, cold_fs, remote_seg_path, local_seg_path, dest_seg_path, local_path, - local_file, remote_files, local_files_with_checksum)); + RETURN_IF_ERROR(download_and_upload_one_cold_file(dest_fs, cold_fs, remote_seg_path, + local_seg_path, dest_seg_path, local_path, + local_file, remote_files)); } if (!have_inverted_index) { @@ -231,9 +225,9 @@ static Status upload_remote_cold_rowset(io::RemoteFileSystem& dest_fs, int64_t t std::string local_seg_path = fmt::format("{}/{}", local_path, index_file); std::string dest_seg_path = fmt::format("{}/{}", dest_path, index_file); - RETURN_IF_ERROR(download_and_upload_one_cold_file( - dest_fs, cold_fs, remote_index_path, local_seg_path, dest_seg_path, local_path, - index_file, remote_files, local_files_with_checksum)); + RETURN_IF_ERROR(download_and_upload_one_cold_file(dest_fs, cold_fs, remote_index_path, + local_seg_path, dest_seg_path, local_path, + index_file, remote_files)); } return res; } @@ -244,8 +238,7 @@ static Status upload_remote_cold_rowset(io::RemoteFileSystem& dest_fs, int64_t t */ static Status upload_remote_cold_file(io::RemoteFileSystem& dest_fs, int64_t tablet_id, const std::string& local_path, const std::string& dest_path, - std::map& remote_files, - std::vector* local_files_with_checksum) { + std::map& remote_files) { Status res = Status::OK(); std::string hdr_file = local_path + "/" + std::to_string(tablet_id) + ".hdr"; auto tablet_meta = std::make_shared(); @@ -276,9 +269,9 @@ static Status upload_remote_cold_file(io::RemoteFileSystem& dest_fs, int64_t tab have_inverted_index = rowset_meta->tablet_schema()->has_inverted_index(); if (segments > 0 && !rowset_meta->is_local()) { - RETURN_IF_ERROR(upload_remote_cold_rowset( - dest_fs, tablet_id, local_path, dest_path, storage_resource, rowset_id, - segments, have_inverted_index, remote_files, local_files_with_checksum)); + RETURN_IF_ERROR(upload_remote_cold_rowset(dest_fs, tablet_id, local_path, dest_path, + storage_resource, rowset_id, segments, + have_inverted_index, remote_files)); } } @@ -335,9 +328,9 @@ Status SnapshotLoader::upload(const std::map& src_to_d TTaskType::type::UPLOAD)); bool need_upload = false; std::string md5sum; - RETURN_IF_ERROR(check_need_upload(src_path, local_file, remote_files, - &local_files_with_checksum, &md5sum, &need_upload)); - + RETURN_IF_ERROR( + check_need_upload(src_path, local_file, remote_files, &md5sum, &need_upload)); + local_files_with_checksum.push_back(local_file + "." + md5sum); if (!need_upload) { VLOG_CRITICAL << "file exist in remote path, no need to upload: " << local_file; continue; @@ -350,8 +343,8 @@ Status SnapshotLoader::upload(const std::map& src_to_d } // end for each tablet's local files // 2.4. upload cooldown data files - RETURN_IF_ERROR(upload_remote_cold_file(*_remote_fs, tablet_id, src_path, dest_path, - remote_files, &local_files_with_checksum)); + RETURN_IF_ERROR( + upload_remote_cold_file(*_remote_fs, tablet_id, src_path, dest_path, remote_files)); tablet_files->emplace(tablet_id, local_files_with_checksum); finished_num++; From 626b7d6444c730226298704cc817dcefb43fca43 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Thu, 5 Dec 2024 20:06:06 +0800 Subject: [PATCH 14/15] fix testcase timeout --- .../doris/service/FrontendServiceImpl.java | 13 ++++-- .../test_backup_restore_cold_data.groovy | 46 +++++++++---------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 95bd9de911d5ea..0db95b79815e65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -28,6 +28,7 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.backup.AbstractJob; import org.apache.doris.backup.BackupJob; import org.apache.doris.backup.Snapshot; import org.apache.doris.catalog.AutoIncrementGenerator; @@ -345,11 +346,15 @@ public TConfirmUnusedRemoteFilesResult confirmUnusedRemoteFiles(TConfirmUnusedRe return; } - BackupJob backupJob = (BackupJob) Env.getCurrentEnv().getBackupHandler().getJob(tabletMeta.getDbId()); + AbstractJob job = Env.getCurrentEnv().getBackupHandler().getJob(tabletMeta.getDbId()); - if (!backupJob.isDone() && backupJob.getBackupMeta().getTable((tabletMeta.getTableId())) != null) { - LOG.warn("Backup is running on this tablet {} ", info.tablet_id); - return; + if (job != null && job instanceof BackupJob) { + BackupJob backupJob = (BackupJob) job; + if (!backupJob.isDone() + && backupJob.getBackupMeta().getTable((tabletMeta.getTableId())) != null) { + LOG.warn("Backup is running on this tablet {} ", info.tablet_id); + return; + } } Tablet tablet; diff --git a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy index b99e3e86b47f4d..bca2d05b226ed6 100644 --- a/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy +++ b/regression-test/suites/backup_restore/test_backup_restore_cold_data.groovy @@ -139,9 +139,9 @@ suite("test_backup_cooldown", "backup_cooldown_data") { sqlResult = result[0][5].toString(); int count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after 30 mins") } Thread.sleep(5000) @@ -195,9 +195,9 @@ suite("test_backup_cooldown", "backup_cooldown_data") { sqlResult = result[0][5].toString(); count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after r0 mins") } Thread.sleep(5000) @@ -340,9 +340,9 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { sqlResult = result[0][5].toString(); int count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after 30 mins") } Thread.sleep(5000) @@ -487,9 +487,9 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { sqlResult = result[0][5].toString(); count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after 30 mins") } Thread.sleep(5000) @@ -523,9 +523,9 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { sqlResult = result[0][5].toString(); count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after 30 mins") } Thread.sleep(5000) @@ -596,9 +596,9 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { sqlResult = result[0][5].toString(); count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after 30 mins") } Thread.sleep(5000) @@ -635,9 +635,9 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { sqlResult = result[0][5].toString(); count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after 30 mins") } Thread.sleep(5000) @@ -867,9 +867,9 @@ suite("test_backup_cooldown_1", "backup_cooldown_data") { sqlResult = result[0][5].toString(); count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after 30 mins") } Thread.sleep(5000) @@ -1044,9 +1044,9 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { sqlResult = result[0][5].toString(); int count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after 30 mins") } Thread.sleep(5000) @@ -1287,9 +1287,9 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { sqlResult = result[0][5].toString(); count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after 30 mins") } Thread.sleep(5000) @@ -1430,9 +1430,9 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { sqlResult = result[0][5].toString(); count = 0; while (sqlResult.contains("0.00")) { - if (++count >= 120) { // 10min + if (++count >= 360) { // 30min logger.error('cooldown task is timeouted') - throw new Exception("cooldown task is timeouted after 10 mins") + throw new Exception("cooldown task is timeouted after 30 mins") } Thread.sleep(5000) @@ -1503,4 +1503,4 @@ suite("test_backup_cooldown_2", "backup_cooldown_data") { try_sql """ drop resource ${resource_name2}; """ -} \ No newline at end of file +} From 8a6c45451ea029ab6fbed515ae40ea832f48e895 Mon Sep 17 00:00:00 2001 From: ayuanzhang Date: Wed, 11 Dec 2024 16:29:46 +0800 Subject: [PATCH 15/15] save storage policy to jobinfo instead of BackupMeta --- .../apache/doris/common/FeMetaVersion.java | 4 +--- .../apache/doris/backup/BackupJobInfo.java | 13 +++------- .../org/apache/doris/backup/BackupMeta.java | 17 ------------- .../org/apache/doris/backup/RestoreJob.java | 24 +++++++------------ 4 files changed, 12 insertions(+), 46 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java index dc16162bca45fd..f1613f5e7cccd0 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -103,10 +103,8 @@ public final class FeMetaVersion { public static final int VERSION_140 = 140; - // For BackupMeta storage policy - public static final int VERSION_141 = 141; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_141; + public static final int VERSION_CURRENT = VERSION_140; // all logs meta version should >= the minimum version, so that we could remove many if clause, for example // if (FE_METAVERSION < VERSION_94) ... diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java index 09b4c8fdf08c82..ca730f0c8cae7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java @@ -340,7 +340,7 @@ public static class BriefBackupJobInfo { @SerializedName("s3_resource_list") public List s3ResourceList = Lists.newArrayList(); @SerializedName("storage_policy_list") - public List storagePolicyList = Lists.newArrayList(); + public List storagePolicyList = Lists.newArrayList(); public static BriefBackupJobInfo fromBackupJobInfo(BackupJobInfo backupJobInfo) { BriefBackupJobInfo briefBackupJobInfo = new BriefBackupJobInfo(); @@ -381,7 +381,7 @@ public static class NewBackupObjects { @SerializedName("s3_resources") public List s3Resources = Lists.newArrayList(); @SerializedName("storage_policy") - public List storagePolicies = Lists.newArrayList(); + public List storagePolicies = Lists.newArrayList(); } public static class BackupOlapTableInfo { @@ -505,11 +505,6 @@ public static class BackupS3ResourceInfo { public String name; } - public static class BackupStoragePolicyInfo { - @SerializedName("name") - public String name; - } - // eg: __db_10001/__tbl_10002/__part_10003/__idx_10002/__10004 public String getFilePath(String db, String tbl, String part, String idx, long tabletId) { if (!db.equalsIgnoreCase(dbName)) { @@ -721,9 +716,7 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db // storage policies Collection storagePolicies = backupMeta.getStoragePolicyNameMap().values(); for (StoragePolicy storagePolicy : storagePolicies) { - BackupStoragePolicyInfo backupStoragePolicyInfo = new BackupStoragePolicyInfo(); - backupStoragePolicyInfo.name = storagePolicy.getName(); - jobInfo.newBackupObjects.storagePolicies.add(backupStoragePolicyInfo); + jobInfo.newBackupObjects.storagePolicies.add(storagePolicy.clone()); } return jobInfo; diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java index 82323d97b6abe8..4947e9eeed74cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java @@ -54,7 +54,6 @@ public class BackupMeta implements Writable, GsonPostProcessable { @SerializedName(value = "resourceNameMap") private Map resourceNameMap = Maps.newHashMap(); // storagePolicy name -> resource - @SerializedName(value = "storagePolicyNameMap") private Map storagePolicyNameMap = Maps.newHashMap(); private BackupMeta() { @@ -108,10 +107,6 @@ public Resource getResource(String resourceName) { return resourceNameMap.get(resourceName); } - public StoragePolicy getStoragePolicy(String policyName) { - return storagePolicyNameMap.get(policyName); - } - public Table getTable(Long tblId) { return tblIdMap.get(tblId); } @@ -168,10 +163,6 @@ public void write(DataOutput out) throws IOException { for (Resource resource : resourceNameMap.values()) { resource.write(out); } - out.writeInt(storagePolicyNameMap.size()); - for (StoragePolicy storagePolicy : storagePolicyNameMap.values()) { - storagePolicy.write(out); - } } @Deprecated @@ -187,14 +178,6 @@ public void readFields(DataInput in) throws IOException { Resource resource = Resource.read(in); resourceNameMap.put(resource.getName(), resource); } - - if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_141) { - size = in.readInt(); - for (int i = 0; i < size; i++) { - StoragePolicy policy = StoragePolicy.read(in); - storagePolicyNameMap.put(policy.getName(), policy); - } - } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index b5bbf8cd62e47a..51348345b1d5ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -700,8 +700,8 @@ private void checkAndPrepareMeta() { } } - for (BackupJobInfo.BackupStoragePolicyInfo backupStoragePolicyInfo : jobInfo.newBackupObjects.storagePolicies) { - String backupStoragePoliceName = backupStoragePolicyInfo.name; + for (StoragePolicy backupStoragePolicy : jobInfo.newBackupObjects.storagePolicies) { + String backupStoragePoliceName = backupStoragePolicy.getName(); Optional localPolicy = Env.getCurrentEnv().getPolicyMgr().findPolicy(backupStoragePoliceName, PolicyTypeEnum.STORAGE); if (localPolicy.isPresent() && localPolicy.get().getType() != PolicyTypeEnum.STORAGE) { @@ -1390,21 +1390,19 @@ private void checkAndRestoreStoragePolicies() { return; } PolicyMgr policyMgr = Env.getCurrentEnv().getPolicyMgr(); - for (BackupJobInfo.BackupStoragePolicyInfo backupStoragePolicyInfo : jobInfo.newBackupObjects.storagePolicies) { - String backupStoragePoliceName = backupStoragePolicyInfo.name; + for (StoragePolicy backupStoragePolicy : jobInfo.newBackupObjects.storagePolicies) { + String backupStoragePoliceName = backupStoragePolicy.getName(); Optional localPolicy = policyMgr.findPolicy(backupStoragePoliceName, PolicyTypeEnum.STORAGE); - StoragePolicy backupStoargePolicy = backupMeta.getStoragePolicy(backupStoragePoliceName); - // use specified storageResource if (storageResource != null) { - backupStoargePolicy.setStorageResource(storageResource); + backupStoragePolicy.setStorageResource(storageResource); } if (localPolicy.isPresent()) { StoragePolicy localStoargePolicy = (StoragePolicy) localPolicy.get(); // storage policy name and resource name should be same if (localStoargePolicy.getSignature(BackupHandler.SIGNATURE_VERSION) - != backupStoargePolicy.getSignature(BackupHandler.SIGNATURE_VERSION)) { + != backupStoragePolicy.getSignature(BackupHandler.SIGNATURE_VERSION)) { status = new Status(ErrCode.COMMON_ERROR, "Storage policy " + jobInfo.getAliasByOriginNameIfSet(backupStoragePoliceName) + " already exist but with different properties"); @@ -1414,11 +1412,11 @@ private void checkAndRestoreStoragePolicies() { } else { // restore storage policy try { - policyMgr.createStoragePolicy(backupStoargePolicy); + policyMgr.createStoragePolicy(backupStoragePolicy); } catch (Exception e) { LOG.error("restore storage policy fail should not happen", e); } - storagePolicies.add(backupStoargePolicy); + storagePolicies.add(backupStoragePolicy); } } } @@ -2818,12 +2816,6 @@ private void readOthers(DataInput in) throws IOException { restoredResources.add(Resource.read(in)); } - // restored storage policy - size = in.readInt(); - for (int i = 0; i < size; i++) { - storagePolicies.add(StoragePolicy.read(in)); - } - // read properties size = in.readInt(); for (int i = 0; i < size; i++) {