diff --git a/be/src/cloud/cloud_backend_service.cpp b/be/src/cloud/cloud_backend_service.cpp index 265e6c44aac9ab..d260e256afc450 100644 --- a/be/src/cloud/cloud_backend_service.cpp +++ b/be/src/cloud/cloud_backend_service.cpp @@ -36,6 +36,9 @@ namespace doris { +bvar::Adder g_file_cache_warm_up_cache_async_submitted_segment_num( + "file_cache_warm_up_cache_async_submitted_segment_num"); + CloudBackendService::CloudBackendService(CloudStorageEngine& engine, ExecEnv* exec_env) : BaseBackendService(exec_env), _engine(engine) {} @@ -93,8 +96,15 @@ void CloudBackendService::warm_up_tablets(TWarmUpTabletsResponse& response, LOG_INFO("receive the warm up request.") .tag("request_type", "SET_JOB") .tag("job_id", request.job_id); - st = manager.check_and_set_job_id(request.job_id); - if (!st) { + if (request.__isset.event) { + st = manager.set_event(request.job_id, request.event); + if (st.ok()) { + break; + } + } else { + st = manager.check_and_set_job_id(request.job_id); + } + if (!st.ok()) { LOG_WARNING("SET_JOB failed.").error(st); break; } @@ -140,7 +150,11 @@ void CloudBackendService::warm_up_tablets(TWarmUpTabletsResponse& response, LOG_INFO("receive the warm up request.") .tag("request_type", "CLEAR_JOB") .tag("job_id", request.job_id); - st = manager.clear_job(request.job_id); + if (request.__isset.event) { + st = manager.set_event(request.job_id, request.event, /* clear: */ true); + } else { + st = manager.clear_job(request.job_id); + } break; } default: @@ -179,6 +193,8 @@ void CloudBackendService::warm_up_cache_async(TWarmUpCacheAsyncResponse& respons PGetFileCacheMetaResponse brpc_response; brpc_stub->get_file_cache_meta_by_tablet_id(&cntl, &brpc_request, &brpc_response, nullptr); if (!cntl.Failed()) { + g_file_cache_warm_up_cache_async_submitted_segment_num + << brpc_response.file_cache_block_metas().size(); _engine.file_cache_block_downloader().submit_download_task( std::move(*brpc_response.mutable_file_cache_block_metas())); } else { diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 66e089c22e9d53..72267252aa63b8 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -19,7 +19,9 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet_mgr.h" +#include "cloud/config.h" #include "io/cache/block_file_cache.h" +#include "io/cache/block_file_cache_downloader.h" #include "io/cache/block_file_cache_factory.h" namespace doris { @@ -105,4 +107,268 @@ void CloudInternalServiceImpl::get_file_cache_meta_by_tablet_id( } } +bvar::Adder g_file_cache_event_driven_warm_up_submitted_segment_num( + "file_cache_event_driven_warm_up_submitted_segment_num"); +bvar::Adder g_file_cache_event_driven_warm_up_finished_segment_num( + "file_cache_event_driven_warm_up_finished_segment_num"); +bvar::Adder g_file_cache_event_driven_warm_up_failed_segment_num( + "file_cache_event_driven_warm_up_failed_segment_num"); +bvar::Adder g_file_cache_event_driven_warm_up_submitted_segment_size( + "file_cache_event_driven_warm_up_submitted_segment_size"); +bvar::Adder g_file_cache_event_driven_warm_up_finished_segment_size( + "file_cache_event_driven_warm_up_finished_segment_size"); +bvar::Adder g_file_cache_event_driven_warm_up_failed_segment_size( + "file_cache_event_driven_warm_up_failed_segment_size"); +bvar::Adder g_file_cache_event_driven_warm_up_submitted_index_num( + "file_cache_event_driven_warm_up_submitted_index_num"); +bvar::Adder g_file_cache_event_driven_warm_up_finished_index_num( + "file_cache_event_driven_warm_up_finished_index_num"); +bvar::Adder g_file_cache_event_driven_warm_up_failed_index_num( + "file_cache_event_driven_warm_up_failed_index_num"); +bvar::Adder g_file_cache_event_driven_warm_up_submitted_index_size( + "file_cache_event_driven_warm_up_submitted_index_size"); +bvar::Adder g_file_cache_event_driven_warm_up_finished_index_size( + "file_cache_event_driven_warm_up_finished_index_size"); +bvar::Adder g_file_cache_event_driven_warm_up_failed_index_size( + "file_cache_event_driven_warm_up_failed_index_size"); +bvar::Status g_file_cache_warm_up_rowset_last_handle_unix_ts( + "file_cache_warm_up_rowset_last_handle_unix_ts", 0); +bvar::Status g_file_cache_warm_up_rowset_last_finish_unix_ts( + "file_cache_warm_up_rowset_last_finish_unix_ts", 0); +bvar::LatencyRecorder g_file_cache_warm_up_rowset_latency("file_cache_warm_up_rowset_latency"); +bvar::LatencyRecorder g_file_cache_warm_up_rowset_request_to_handle_latency( + "file_cache_warm_up_rowset_request_to_handle_latency"); +bvar::LatencyRecorder g_file_cache_warm_up_rowset_handle_to_finish_latency( + "file_cache_warm_up_rowset_handle_to_finish_latency"); +bvar::Adder g_file_cache_warm_up_rowset_slow_count( + "file_cache_warm_up_rowset_slow_count"); +bvar::Adder g_file_cache_warm_up_rowset_request_to_handle_slow_count( + "file_cache_warm_up_rowset_request_to_handle_slow_count"); +bvar::Adder g_file_cache_warm_up_rowset_handle_to_finish_slow_count( + "file_cache_warm_up_rowset_handle_to_finish_slow_count"); + +void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* controller + [[maybe_unused]], + const PWarmUpRowsetRequest* request, + PWarmUpRowsetResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard closure_guard(done); + for (auto& rs_meta_pb : request->rowset_metas()) { + RowsetMeta rs_meta; + rs_meta.init_from_pb(rs_meta_pb); + auto storage_resource = rs_meta.remote_storage_resource(); + if (!storage_resource) { + LOG(WARNING) << storage_resource.error(); + continue; + } + int64_t tablet_id = rs_meta.tablet_id(); + auto res = _engine.tablet_mgr().get_tablet(tablet_id); + if (!res.has_value()) { + LOG_WARNING("Warm up error ").tag("tablet_id", tablet_id).error(res.error()); + continue; + } + auto tablet = res.value(); + auto tablet_meta = tablet->tablet_meta(); + + int64_t handle_ts = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + g_file_cache_warm_up_rowset_last_handle_unix_ts.set_value(handle_ts); + int64_t request_ts = request->has_unix_ts_us() ? request->unix_ts_us() : 0; + g_file_cache_warm_up_rowset_request_to_handle_latency << (handle_ts - request_ts); + if (request_ts > 0 && handle_ts - request_ts > config::warm_up_rowset_slow_log_ms * 1000) { + g_file_cache_warm_up_rowset_request_to_handle_slow_count << 1; + LOG(INFO) << "warm up rowset (request to handle) took " << handle_ts - request_ts + << " us, tablet_id: " << rs_meta.tablet_id() + << ", rowset_id: " << rs_meta.rowset_id().to_string(); + } + int64_t expiration_time = + tablet_meta->ttl_seconds() == 0 || rs_meta.newest_write_timestamp() <= 0 + ? 0 + : rs_meta.newest_write_timestamp() + tablet_meta->ttl_seconds(); + if (expiration_time <= UnixSeconds()) { + expiration_time = 0; + } + + for (int64_t segment_id = 0; segment_id < rs_meta.num_segments(); segment_id++) { + auto download_done = [=, tablet_id = rs_meta.tablet_id(), + rowset_id = rs_meta.rowset_id().to_string(), + segment_size = rs_meta.segment_file_size(segment_id)](Status st) { + if (st.ok()) { + g_file_cache_event_driven_warm_up_finished_segment_num << 1; + g_file_cache_event_driven_warm_up_finished_segment_size << segment_size; + int64_t now_ts = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + g_file_cache_warm_up_rowset_last_finish_unix_ts.set_value(now_ts); + g_file_cache_warm_up_rowset_latency << (now_ts - request_ts); + g_file_cache_warm_up_rowset_handle_to_finish_latency << (now_ts - handle_ts); + if (request_ts > 0 && + now_ts - request_ts > config::warm_up_rowset_slow_log_ms * 1000) { + g_file_cache_warm_up_rowset_slow_count << 1; + LOG(INFO) << "warm up rowset took " << now_ts - request_ts + << " us, tablet_id: " << tablet_id << ", rowset_id: " << rowset_id + << ", segment_id: " << segment_id; + } + if (now_ts - handle_ts > config::warm_up_rowset_slow_log_ms * 1000) { + g_file_cache_warm_up_rowset_handle_to_finish_slow_count << 1; + LOG(INFO) << "warm up rowset (handle to finish) took " << now_ts - handle_ts + << " us, tablet_id: " << tablet_id << ", rowset_id: " << rowset_id + << ", segment_id: " << segment_id; + } + } else { + g_file_cache_event_driven_warm_up_failed_segment_num << 1; + g_file_cache_event_driven_warm_up_failed_segment_size << segment_size; + LOG(WARNING) << "download segment failed, tablet_id: " << tablet_id + << " rowset_id: " << rowset_id << ", error: " << st; + } + }; + + io::DownloadFileMeta download_meta { + .path = storage_resource.value()->remote_segment_path(rs_meta, segment_id), + .file_size = rs_meta.segment_file_size(segment_id), + .offset = 0, + .download_size = rs_meta.segment_file_size(segment_id), + .file_system = storage_resource.value()->fs, + .ctx = + { + .is_index_data = false, + .expiration_time = expiration_time, + .is_dryrun = + config::enable_reader_dryrun_when_download_file_cache, + }, + .download_done = std::move(download_done), + }; + g_file_cache_event_driven_warm_up_submitted_segment_num << 1; + g_file_cache_event_driven_warm_up_submitted_segment_size + << rs_meta.segment_file_size(segment_id); + _engine.file_cache_block_downloader().submit_download_task(download_meta); + + auto download_inverted_index = [&](std::string index_path, uint64_t idx_size) { + auto storage_resource = rs_meta.remote_storage_resource(); + auto download_done = [=, tablet_id = rs_meta.tablet_id(), + rowset_id = rs_meta.rowset_id().to_string()](Status st) { + if (st.ok()) { + g_file_cache_event_driven_warm_up_finished_index_num << 1; + g_file_cache_event_driven_warm_up_finished_index_size << idx_size; + int64_t now_ts = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + g_file_cache_warm_up_rowset_last_finish_unix_ts.set_value(now_ts); + g_file_cache_warm_up_rowset_latency << (now_ts - request_ts); + g_file_cache_warm_up_rowset_handle_to_finish_latency + << (now_ts - handle_ts); + if (request_ts > 0 && + now_ts - request_ts > config::warm_up_rowset_slow_log_ms * 1000) { + g_file_cache_warm_up_rowset_slow_count << 1; + LOG(INFO) << "warm up rowset took " << now_ts - request_ts + << " us, tablet_id: " << tablet_id + << ", rowset_id: " << rowset_id + << ", segment_id: " << segment_id; + } + if (now_ts - handle_ts > config::warm_up_rowset_slow_log_ms * 1000) { + g_file_cache_warm_up_rowset_handle_to_finish_slow_count << 1; + LOG(INFO) << "warm up rowset (handle to finish) took " + << now_ts - handle_ts << " us, tablet_id: " << tablet_id + << ", rowset_id: " << rowset_id + << ", segment_id: " << segment_id; + } + } else { + g_file_cache_event_driven_warm_up_failed_index_num << 1; + g_file_cache_event_driven_warm_up_failed_index_size << idx_size; + LOG(WARNING) << "download inverted index failed, tablet_id: " << tablet_id + << " rowset_id: " << rowset_id << ", error: " << st; + } + }; + io::DownloadFileMeta download_meta { + .path = io::Path(index_path), + .file_size = static_cast(idx_size), + .file_system = storage_resource.value()->fs, + .ctx = + { + .is_index_data = false, // DORIS-20877 + .expiration_time = expiration_time, + .is_dryrun = config:: + enable_reader_dryrun_when_download_file_cache, + }, + .download_done = std::move(download_done), + }; + g_file_cache_event_driven_warm_up_submitted_index_num << 1; + g_file_cache_event_driven_warm_up_submitted_index_size << idx_size; + _engine.file_cache_block_downloader().submit_download_task(download_meta); + }; + + // inverted index + auto schema_ptr = rs_meta.tablet_schema(); + auto idx_version = schema_ptr->get_inverted_index_storage_format(); + bool has_inverted_index = schema_ptr->has_inverted_index(); + + if (has_inverted_index) { + if (idx_version == InvertedIndexStorageFormatPB::V1) { + auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id); + std::unordered_map index_size_map; + for (const auto& info : inverted_index_info.index_info()) { + if (info.index_file_size() != -1) { + index_size_map[info.index_id()] = info.index_file_size(); + } else { + VLOG_DEBUG << "Invalid index_file_size for segment_id " << segment_id + << ", index_id " << info.index_id(); + } + } + for (const auto& index : schema_ptr->inverted_indexes()) { + auto idx_path = storage_resource.value()->remote_idx_v1_path( + rs_meta, segment_id, index->index_id(), index->get_index_suffix()); + download_inverted_index(idx_path, index_size_map[index->index_id()]); + } + } else { // InvertedIndexStorageFormatPB::V2 + auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id); + int64_t idx_size = 0; + if (inverted_index_info.has_index_size()) { + idx_size = inverted_index_info.index_size(); + } else { + VLOG_DEBUG << "index_size is not set for segment " << segment_id; + } + auto idx_path = + storage_resource.value()->remote_idx_v2_path(rs_meta, segment_id); + download_inverted_index(idx_path, idx_size); + } + } + } + } +} + +bvar::Adder g_file_cache_recycle_cache_finished_segment_num( + "file_cache_recycle_cache_finished_segment_num"); +bvar::Adder g_file_cache_recycle_cache_finished_index_num( + "file_cache_recycle_cache_finished_index_num"); + +void CloudInternalServiceImpl::recycle_cache(google::protobuf::RpcController* controller + [[maybe_unused]], + const PRecycleCacheRequest* request, + PRecycleCacheResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard closure_guard(done); + + if (!config::enable_file_cache) { + return; + } + for (const auto& meta : request->cache_metas()) { + for (int64_t segment_id = 0; segment_id < meta.num_segments(); segment_id++) { + auto file_key = Segment::file_cache_key(meta.rowset_id(), segment_id); + auto* file_cache = io::FileCacheFactory::instance()->get_by_path(file_key); + file_cache->remove_if_cached_async(file_key); + g_file_cache_recycle_cache_finished_segment_num << 1; + } + + // inverted index + for (const auto& file_name : meta.index_file_names()) { + auto file_key = io::BlockFileCache::hash(file_name); + auto* file_cache = io::FileCacheFactory::instance()->get_by_path(file_key); + file_cache->remove_if_cached_async(file_key); + g_file_cache_recycle_cache_finished_index_num << 1; + } + } +} + } // namespace doris diff --git a/be/src/cloud/cloud_internal_service.h b/be/src/cloud/cloud_internal_service.h index db93a82a719e37..59d8739cbf46d6 100644 --- a/be/src/cloud/cloud_internal_service.h +++ b/be/src/cloud/cloud_internal_service.h @@ -40,6 +40,14 @@ class CloudInternalServiceImpl final : public PInternalService { PGetFileCacheMetaResponse* response, google::protobuf::Closure* done) override; + void warm_up_rowset(google::protobuf::RpcController* controller, + const PWarmUpRowsetRequest* request, PWarmUpRowsetResponse* response, + google::protobuf::Closure* done) override; + + void recycle_cache(google::protobuf::RpcController* controller, + const PRecycleCacheRequest* request, PRecycleCacheResponse* response, + google::protobuf::Closure* done) override; + private: CloudStorageEngine& _engine; }; diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index ca179144310f4f..e790103e5bd51b 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -39,6 +39,7 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" +#include "cloud/cloud_warm_up_manager.h" #include "cloud/config.h" #include "cloud/pb_convert.h" #include "cloud/schema_cloud_dictionary_cache.h" @@ -968,7 +969,7 @@ Status CloudMetaMgr::prepare_rowset(const RowsetMeta& rs_meta, const std::string return st; } -Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, const std::string& job_id, +Status CloudMetaMgr::commit_rowset(RowsetMeta& rs_meta, const std::string& job_id, RowsetMetaSharedPtr* existed_rs_meta) { VLOG_DEBUG << "commit rowset, tablet_id: " << rs_meta.tablet_id() << ", rowset_id: " << rs_meta.rowset_id() << " txn_id: " << rs_meta.txn_id(); @@ -1010,6 +1011,8 @@ Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, const std::string& RETURN_IF_ERROR( engine.get_schema_cloud_dictionary_cache().refresh_dict(rs_meta_pb.index_id())); } + auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); + manager.warm_up_rowset(rs_meta); return st; } diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h index 99f409ed6dde6d..a433e645a35dc2 100644 --- a/be/src/cloud/cloud_meta_mgr.h +++ b/be/src/cloud/cloud_meta_mgr.h @@ -73,7 +73,7 @@ class CloudMetaMgr { Status prepare_rowset(const RowsetMeta& rs_meta, const std::string& job_id, std::shared_ptr* existed_rs_meta = nullptr); - Status commit_rowset(const RowsetMeta& rs_meta, const std::string& job_id, + Status commit_rowset(RowsetMeta& rs_meta, const std::string& job_id, std::shared_ptr* existed_rs_meta = nullptr); Status update_tmp_rowset(const RowsetMeta& rs_meta); diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 8151560599d648..a8e9d43f6ca5ba 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -35,6 +35,7 @@ #include "cloud/cloud_meta_mgr.h" #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet_mgr.h" +#include "cloud/cloud_warm_up_manager.h" #include "common/config.h" #include "common/logging.h" #include "io/cache/block_file_cache_downloader.h" @@ -62,6 +63,30 @@ bvar::Adder g_unused_rowsets_count("unused_rowsets_count"); static constexpr int LOAD_INITIATOR_ID = -1; +bvar::Adder g_file_cache_cloud_tablet_submitted_segment_size( + "file_cache_cloud_tablet_submitted_segment_size"); +bvar::Adder g_file_cache_cloud_tablet_submitted_segment_num( + "file_cache_cloud_tablet_submitted_segment_num"); +bvar::Adder g_file_cache_cloud_tablet_submitted_index_size( + "file_cache_cloud_tablet_submitted_index_size"); +bvar::Adder g_file_cache_cloud_tablet_submitted_index_num( + "file_cache_cloud_tablet_submitted_index_num"); +bvar::Adder g_file_cache_cloud_tablet_finished_segment_size( + "file_cache_cloud_tablet_finished_segment_size"); +bvar::Adder g_file_cache_cloud_tablet_finished_segment_num( + "file_cache_cloud_tablet_finished_segment_num"); +bvar::Adder g_file_cache_cloud_tablet_finished_index_size( + "file_cache_cloud_tablet_finished_index_size"); +bvar::Adder g_file_cache_cloud_tablet_finished_index_num( + "file_cache_cloud_tablet_finished_index_num"); + +bvar::Adder g_file_cache_recycle_cached_data_segment_num( + "file_cache_recycle_cached_data_segment_num"); +bvar::Adder g_file_cache_recycle_cached_data_segment_size( + "file_cache_recycle_cached_data_segment_size"); +bvar::Adder g_file_cache_recycle_cached_data_index_num( + "file_cache_recycle_cached_data_index_num"); + CloudTablet::CloudTablet(CloudStorageEngine& engine, TabletMetaSharedPtr tablet_meta) : BaseTablet(std::move(tablet_meta)), _engine(engine) {} @@ -267,48 +292,83 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ ? 0 : rowset_meta->newest_write_timestamp() + _tablet_meta->ttl_seconds(); - // clang-format off + g_file_cache_cloud_tablet_submitted_segment_num << 1; + if (rs->rowset_meta()->segment_file_size(seg_id) > 0) { + g_file_cache_cloud_tablet_submitted_segment_size + << rs->rowset_meta()->segment_file_size(seg_id); + } _engine.file_cache_block_downloader().submit_download_task(io::DownloadFileMeta { - .path = storage_resource.value()->remote_segment_path(*rowset_meta, seg_id), + .path = storage_resource.value()->remote_segment_path(*rowset_meta, + seg_id), .file_size = rs->rowset_meta()->segment_file_size(seg_id), .file_system = storage_resource.value()->fs, .ctx = { .expiration_time = expiration_time, - .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_dryrun = config:: + enable_reader_dryrun_when_download_file_cache, }, - .download_done {}, + .download_done {[](Status st) { + if (!st) { + LOG_WARNING("add rowset warm up error ").error(st); + } + }}, }); - auto download_idx_file = [&](const io::Path& idx_path) { + auto download_idx_file = [&](const io::Path& idx_path, int64_t idx_size) { io::DownloadFileMeta meta { .path = idx_path, - .file_size = -1, + .file_size = idx_size, .file_system = storage_resource.value()->fs, .ctx = { .expiration_time = expiration_time, - .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_dryrun = config:: + enable_reader_dryrun_when_download_file_cache, }, - .download_done {}, + .download_done {[](Status st) { + if (!st) { + LOG_WARNING("add rowset warm up error ").error(st); + } + }}, }; _engine.file_cache_block_downloader().submit_download_task(std::move(meta)); + g_file_cache_cloud_tablet_submitted_index_num << 1; + g_file_cache_cloud_tablet_submitted_index_size << idx_size; }; // clang-format on auto schema_ptr = rowset_meta->tablet_schema(); auto idx_version = schema_ptr->get_inverted_index_storage_format(); if (idx_version == InvertedIndexStorageFormatPB::V1) { + std::unordered_map index_size_map; + auto&& inverted_index_info = rowset_meta->inverted_index_file_info(seg_id); + for (const auto& info : inverted_index_info.index_info()) { + if (info.index_file_size() != -1) { + index_size_map[info.index_id()] = info.index_file_size(); + } else { + VLOG_DEBUG << "Invalid index_file_size for segment_id " << seg_id + << ", index_id " << info.index_id(); + } + } for (const auto& index : schema_ptr->inverted_indexes()) { auto idx_path = storage_resource.value()->remote_idx_v1_path( *rowset_meta, seg_id, index->index_id(), index->get_index_suffix()); - download_idx_file(idx_path); + download_idx_file(idx_path, index_size_map[index->index_id()]); } } else { if (schema_ptr->has_inverted_index()) { + auto&& inverted_index_info = + rowset_meta->inverted_index_file_info(seg_id); + int64_t idx_size = 0; + if (inverted_index_info.has_index_size()) { + idx_size = inverted_index_info.index_size(); + } else { + VLOG_DEBUG << "index_size is not set for segment " << seg_id; + } auto idx_path = storage_resource.value()->remote_idx_v2_path( *rowset_meta, seg_id); - download_idx_file(idx_path); + download_idx_file(idx_path, idx_size); } } } @@ -507,25 +567,53 @@ void CloudTablet::add_unused_rowsets(const std::vector& rowsets } void CloudTablet::remove_unused_rowsets() { - int64_t removed_rowsets_num = 0; int64_t removed_delete_bitmap_num = 0; + std::vector> removed_rowsets; OlapStopWatch watch; - std::lock_guard lock(_gc_mutex); - // 1. remove unused rowsets's cache data and delete bitmap - for (auto it = _unused_rowsets.begin(); it != _unused_rowsets.end();) { - // it->second is std::shared_ptr - auto&& rs = it->second; - if (rs.use_count() > 1) { - LOG(WARNING) << "tablet_id:" << tablet_id() << " rowset: " << rs->rowset_id() << " has " - << rs.use_count() << " references, it cannot be removed"; - ++it; - continue; + + { + std::lock_guard lock(_gc_mutex); + // 1. remove unused rowsets's cache data and delete bitmap + for (auto it = _unused_rowsets.begin(); it != _unused_rowsets.end();) { + auto& rs = it->second; + if (rs.use_count() > 1) { + LOG(WARNING) << "tablet_id:" << tablet_id() << " rowset: " << rs->rowset_id() + << " has " << rs.use_count() << " references, it cannot be removed"; + ++it; + continue; + } + tablet_meta()->remove_rowset_delete_bitmap(rs->rowset_id(), rs->version()); + rs->clear_cache(); + removed_rowsets.push_back(std::move(rs)); + g_unused_rowsets_count << -1; + it = _unused_rowsets.erase(it); + } + } + + { + std::vector rowset_ids; + std::vector num_segments; + std::vector> index_file_names; + + for (auto& rs : removed_rowsets) { + rowset_ids.push_back(rs->rowset_id()); + num_segments.push_back(rs->num_segments()); + auto index_names = rs->get_index_file_names(); + index_file_names.push_back(index_names); + int64_t segment_size_sum = 0; + for (int32_t i = 0; i < rs->num_segments(); i++) { + segment_size_sum += rs->rowset_meta()->segment_file_size(i); + } + g_file_cache_recycle_cached_data_segment_num << rs->num_segments(); + g_file_cache_recycle_cached_data_segment_size << segment_size_sum; + g_file_cache_recycle_cached_data_index_num << index_names.size(); + } + + if (removed_rowsets.size() > 0) { + auto& manager = + ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); + manager.recycle_cache(tablet_id(), rowset_ids, num_segments, index_file_names); } - tablet_meta()->remove_rowset_delete_bitmap(rs->rowset_id(), rs->version()); - rs->clear_cache(); - it = _unused_rowsets.erase(it); - g_unused_rowsets_count << -1; - removed_rowsets_num++; } // 2. remove delete bitmap of pre rowsets @@ -552,7 +640,7 @@ void CloudTablet::remove_unused_rowsets() { LOG(INFO) << "tablet_id=" << tablet_id() << ", unused_rowset size=" << _unused_rowsets.size() << ", unused_delete_bitmap size=" << _unused_delete_bitmap.size() - << ", removed_rowsets_num=" << removed_rowsets_num + << ", removed_rowsets_num=" << removed_rowsets.size() << ", removed_delete_bitmap_num=" << removed_delete_bitmap_num << ", cost(us)=" << watch.get_elapse_time_us(); } @@ -570,14 +658,33 @@ void CloudTablet::clear_cache() { } void CloudTablet::recycle_cached_data(const std::vector& rowsets) { + std::vector rowset_ids; + std::vector num_segments; + std::vector> index_file_names; for (const auto& rs : rowsets) { // rowsets and tablet._rs_version_map each hold a rowset shared_ptr, so at this point, the reference count of the shared_ptr is at least 2. if (rs.use_count() > 2) { LOG(WARNING) << "Rowset " << rs->rowset_id().to_string() << " has " << rs.use_count() << " references. File Cache won't be recycled when query is using it."; - return; + continue; } rs->clear_cache(); + rowset_ids.push_back(rs->rowset_id()); + num_segments.push_back(rs->num_segments()); + auto index_names = rs->get_index_file_names(); + index_file_names.push_back(index_names); + int64_t segment_size_sum = 0; + for (int32_t i = 0; i < rs->num_segments(); i++) { + segment_size_sum += rs->rowset_meta()->segment_file_size(i); + } + g_file_cache_recycle_cached_data_segment_num << rs->num_segments(); + g_file_cache_recycle_cached_data_segment_size << segment_size_sum; + g_file_cache_recycle_cached_data_index_num << index_names.size(); + } + if (!rowsets.empty()) { + auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); + manager.recycle_cache(rowsets.front()->rowset_meta()->tablet_id(), rowset_ids, num_segments, + index_file_names); } } diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 6851790523b897..66e605cb17c81e 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -258,8 +258,6 @@ class CloudTablet final : public BaseTablet { void build_tablet_report_info(TTabletInfo* tablet_info); - static void recycle_cached_data(const std::vector& rowsets); - // check that if the delete bitmap in delete bitmap cache has the same cardinality with the expected_delete_bitmap's Status check_delete_bitmap_cache(int64_t txn_id, DeleteBitmap* expected_delete_bitmap) override; @@ -273,6 +271,8 @@ class CloudTablet final : public BaseTablet { void add_unused_rowsets(const std::vector& rowsets); void remove_unused_rowsets(); + static void recycle_cached_data(const std::vector& rowsets); + private: // FIXME(plat1ko): No need to record base size if rowsets are ordered by version void update_base_size(const Rowset& rs); diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index d8bce097465dde..15c346e465dc0a 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -23,17 +23,53 @@ #include #include +#include "bvar/bvar.h" #include "cloud/cloud_tablet_mgr.h" +#include "cloud/config.h" #include "common/logging.h" #include "io/cache/block_file_cache_downloader.h" #include "olap/rowset/beta_rowset.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" #include "olap/tablet.h" +#include "runtime/client_cache.h" #include "runtime/exec_env.h" +#include "util/brpc_client_cache.h" // BrpcClientCache +#include "util/thrift_rpc_helper.h" #include "util/time.h" namespace doris { +bvar::Adder g_file_cache_event_driven_warm_up_requested_segment_size( + "file_cache_event_driven_warm_up_requested_segment_size"); +bvar::Adder g_file_cache_event_driven_warm_up_requested_segment_num( + "file_cache_event_driven_warm_up_requested_segment_num"); +bvar::Adder g_file_cache_event_driven_warm_up_requested_index_size( + "file_cache_event_driven_warm_up_requested_index_size"); +bvar::Adder g_file_cache_event_driven_warm_up_requested_index_num( + "file_cache_event_driven_warm_up_requested_index_num"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_segment_size( + "file_cache_once_or_periodic_warm_up_submitted_segment_size"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_segment_num( + "file_cache_once_or_periodic_warm_up_submitted_segment_num"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_index_size( + "file_cache_once_or_periodic_warm_up_submitted_index_size"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_index_num( + "file_cache_once_or_periodic_warm_up_submitted_index_num"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_finished_segment_size( + "file_cache_once_or_periodic_warm_up_finished_segment_size"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_finished_segment_num( + "file_cache_once_or_periodic_warm_up_finished_segment_num"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_finished_index_size( + "file_cache_once_or_periodic_warm_up_finished_index_size"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_finished_index_num( + "file_cache_once_or_periodic_warm_up_finished_index_num"); +bvar::Adder g_file_cache_recycle_cache_requested_segment_num( + "file_cache_recycle_cache_requested_segment_num"); +bvar::Adder g_file_cache_recycle_cache_requested_index_num( + "file_cache_recycle_cache_requested_index_num"); +bvar::Status g_file_cache_warm_up_rowset_last_call_unix_ts( + "file_cache_warm_up_rowset_last_call_unix_ts", 0); + CloudWarmUpManager::CloudWarmUpManager(CloudStorageEngine& engine) : _engine(engine) { _download_thread = std::thread(&CloudWarmUpManager::handle_jobs, this); } @@ -70,7 +106,10 @@ void CloudWarmUpManager::handle_jobs() { _cond.wait(lock); } if (_closed) break; - cur_job = _pending_job_metas.front(); + if (!_pending_job_metas.empty()) { + cur_job = _pending_job_metas.front(); + _pending_job_metas.pop_front(); + } } if (!cur_job) { @@ -113,7 +152,11 @@ void CloudWarmUpManager::handle_jobs() { } wait->add_count(); - // clang-format off + g_file_cache_once_or_periodic_warm_up_submitted_segment_num << 1; + if (rs->segment_file_size(seg_id) > 0) { + g_file_cache_once_or_periodic_warm_up_submitted_segment_size + << rs->segment_file_size(seg_id); + } _engine.file_cache_block_downloader().submit_download_task(io::DownloadFileMeta { .path = storage_resource.value()->remote_segment_path(*rs, seg_id), .file_size = rs->segment_file_size(seg_id), @@ -121,7 +164,8 @@ void CloudWarmUpManager::handle_jobs() { .ctx = { .expiration_time = expiration_time, - .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_dryrun = config:: + enable_reader_dryrun_when_download_file_cache, }, .download_done = [wait](Status st) { @@ -132,15 +176,16 @@ void CloudWarmUpManager::handle_jobs() { }, }); - auto download_idx_file = [&](const io::Path& idx_path) { + auto download_idx_file = [&](const io::Path& idx_path, int64_t idx_size) { io::DownloadFileMeta meta { .path = idx_path, - .file_size = -1, + .file_size = idx_size, .file_system = storage_resource.value()->fs, .ctx = { .expiration_time = expiration_time, - .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_dryrun = config:: + enable_reader_dryrun_when_download_file_cache, }, .download_done = [wait](Status st) { @@ -151,23 +196,42 @@ void CloudWarmUpManager::handle_jobs() { }, }; // clang-format on + g_file_cache_once_or_periodic_warm_up_submitted_index_num << 1; + g_file_cache_once_or_periodic_warm_up_submitted_index_size << idx_size; _engine.file_cache_block_downloader().submit_download_task(std::move(meta)); }; auto schema_ptr = rs->tablet_schema(); auto idx_version = schema_ptr->get_inverted_index_storage_format(); if (idx_version == InvertedIndexStorageFormatPB::V1) { + auto&& inverted_index_info = rs->inverted_index_file_info(seg_id); + std::unordered_map index_size_map; + for (const auto& info : inverted_index_info.index_info()) { + if (info.index_file_size() != -1) { + index_size_map[info.index_id()] = info.index_file_size(); + } else { + VLOG_DEBUG << "Invalid index_file_size for segment_id " << seg_id + << ", index_id " << info.index_id(); + } + } for (const auto& index : schema_ptr->inverted_indexes()) { wait->add_count(); auto idx_path = storage_resource.value()->remote_idx_v1_path( *rs, seg_id, index->index_id(), index->get_index_suffix()); - download_idx_file(idx_path); + download_idx_file(idx_path, index_size_map[index->index_id()]); } } else { if (schema_ptr->has_inverted_index()) { + auto&& inverted_index_info = rs->inverted_index_file_info(seg_id); + int64_t idx_size = 0; + if (inverted_index_info.has_index_size()) { + idx_size = inverted_index_info.index_size(); + } else { + VLOG_DEBUG << "index_size is not set for segment " << seg_id; + } wait->add_count(); auto idx_path = storage_resource.value()->remote_idx_v2_path(*rs, seg_id); - download_idx_file(idx_path); + download_idx_file(idx_path, idx_size); } } } @@ -175,13 +239,12 @@ void CloudWarmUpManager::handle_jobs() { timespec time; time.tv_sec = UnixSeconds() + WAIT_TIME_SECONDS; if (!wait->timed_wait(time)) { - LOG_WARNING("Warm up tablet {} take a long time", tablet_meta->tablet_id()); + LOG_WARNING("Warm up {} tablets take a long time", cur_job->tablet_ids.size()); } } { std::unique_lock lock(_mtx); _finish_job.push_back(cur_job); - _pending_job_metas.pop_front(); } } #endif @@ -276,4 +339,255 @@ Status CloudWarmUpManager::clear_job(int64_t job_id) { return st; } +Status CloudWarmUpManager::set_event(int64_t job_id, TWarmUpEventType::type event, bool clear) { + DBUG_EXECUTE_IF("CloudWarmUpManager.set_event.ignore_all", { + LOG(INFO) << "Ignore set_event request, job_id=" << job_id << ", event=" << event + << ", clear=" << clear; + return Status::OK(); + }); + std::lock_guard lock(_mtx); + Status st = Status::OK(); + if (event == TWarmUpEventType::type::LOAD) { + if (clear) { + _tablet_replica_cache.erase(job_id); + LOG(INFO) << "Clear event driven sync, job_id=" << job_id << ", event=" << event; + } else if (!_tablet_replica_cache.contains(job_id)) { + static_cast(_tablet_replica_cache[job_id]); + LOG(INFO) << "Set event driven sync, job_id=" << job_id << ", event=" << event; + } + } else { + st = Status::InternalError("The event {} is not supported yet", event); + } + return st; +} + +std::vector CloudWarmUpManager::get_replica_info(int64_t tablet_id) { + std::vector replicas; + std::vector cancelled_jobs; + std::lock_guard lock(_mtx); + for (auto& [job_id, cache] : _tablet_replica_cache) { + auto it = cache.find(tablet_id); + if (it != cache.end()) { + // check ttl expire + auto now = std::chrono::steady_clock::now(); + auto sec = std::chrono::duration_cast(now - it->second.first); + if (sec.count() < config::warmup_tablet_replica_info_cache_ttl_sec) { + replicas.push_back(it->second.second); + LOG(INFO) << "get_replica_info: cache hit, tablet_id=" << tablet_id + << ", job_id=" << job_id; + continue; + } else { + LOG(INFO) << "get_replica_info: cache expired, tablet_id=" << tablet_id + << ", job_id=" << job_id; + cache.erase(it); + } + } + LOG(INFO) << "get_replica_info: cache miss, tablet_id=" << tablet_id + << ", job_id=" << job_id; + + ClusterInfo* cluster_info = ExecEnv::GetInstance()->cluster_info(); + if (cluster_info == nullptr) { + LOG(WARNING) << "get_replica_info: have not get FE Master heartbeat yet, job_id=" + << job_id; + continue; + } + TNetworkAddress master_addr = cluster_info->master_fe_addr; + if (master_addr.hostname == "" || master_addr.port == 0) { + LOG(WARNING) << "get_replica_info: have not get FE Master heartbeat yet, job_id=" + << job_id; + continue; + } + + TGetTabletReplicaInfosRequest request; + TGetTabletReplicaInfosResult result; + request.warm_up_job_id = job_id; + request.__isset.warm_up_job_id = true; + request.tablet_ids.emplace_back(tablet_id); + Status rpc_st = ThriftRpcHelper::rpc( + master_addr.hostname, master_addr.port, + [&request, &result](FrontendServiceConnection& client) { + client->getTabletReplicaInfos(result, request); + }); + + if (!rpc_st.ok()) { + LOG(WARNING) << "get_replica_info: rpc failed error=" << rpc_st + << ", tablet id=" << tablet_id << ", job_id=" << job_id; + continue; + } + + auto st = Status::create(result.status); + if (!st.ok()) { + if (st.is()) { + LOG(INFO) << "get_replica_info: warm up job cancelled, tablet_id=" << tablet_id + << ", job_id=" << job_id; + cancelled_jobs.push_back(job_id); + } else { + LOG(WARNING) << "get_replica_info: failed status=" << st + << ", tablet id=" << tablet_id << ", job_id=" << job_id; + } + continue; + } + VLOG_DEBUG << "get_replica_info: got " << result.tablet_replica_infos.size() + << " tablets, tablet id=" << tablet_id << ", job_id=" << job_id; + + for (const auto& it : result.tablet_replica_infos) { + auto tablet_id = it.first; + VLOG_DEBUG << "get_replica_info: got " << it.second.size() + << " replica_infos, tablet id=" << tablet_id << ", job_id=" << job_id; + for (const auto& replica : it.second) { + cache[tablet_id] = std::make_pair(std::chrono::steady_clock::now(), replica); + replicas.push_back(replica); + LOG(INFO) << "get_replica_info: cache add, tablet_id=" << tablet_id + << ", job_id=" << job_id; + } + } + } + for (auto job_id : cancelled_jobs) { + LOG(INFO) << "get_replica_info: erasing cancelled job, job_id=" << job_id; + _tablet_replica_cache.erase(job_id); + } + VLOG_DEBUG << "get_replica_info: return " << replicas.size() + << " replicas, tablet id=" << tablet_id; + return replicas; +} + +void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta) { + auto replicas = get_replica_info(rs_meta.tablet_id()); + if (replicas.empty()) { + LOG(INFO) << "There is no need to warmup tablet=" << rs_meta.tablet_id() + << ", skipping rowset=" << rs_meta.rowset_id().to_string(); + return; + } + int64_t now_ts = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + g_file_cache_warm_up_rowset_last_call_unix_ts.set_value(now_ts); + + PWarmUpRowsetRequest request; + request.add_rowset_metas()->CopyFrom(rs_meta.get_rowset_pb()); + request.set_unix_ts_us(now_ts); + for (auto& replica : replicas) { + // send sync request + std::string host = replica.host; + auto dns_cache = ExecEnv::GetInstance()->dns_cache(); + if (dns_cache == nullptr) { + LOG(WARNING) << "DNS cache is not initialized, skipping hostname resolve"; + } else if (!is_valid_ip(replica.host)) { + Status status = dns_cache->get(replica.host, &host); + if (!status.ok()) { + LOG(WARNING) << "failed to get ip from host " << replica.host << ": " + << status.to_string(); + return; + } + } + std::string brpc_addr = get_host_port(host, replica.brpc_port); + Status st = Status::OK(); + std::shared_ptr brpc_stub = + ExecEnv::GetInstance()->brpc_internal_client_cache()->get_new_client_no_cache( + brpc_addr); + if (!brpc_stub) { + st = Status::RpcError("Address {} is wrong", brpc_addr); + continue; + } + + // update metrics + auto schema_ptr = rs_meta.tablet_schema(); + bool has_inverted_index = schema_ptr->has_inverted_index(); + auto idx_version = schema_ptr->get_inverted_index_storage_format(); + for (int64_t segment_id = 0; segment_id < rs_meta.num_segments(); segment_id++) { + g_file_cache_event_driven_warm_up_requested_segment_num << 1; + g_file_cache_event_driven_warm_up_requested_segment_size + << rs_meta.segment_file_size(segment_id); + + if (has_inverted_index) { + if (idx_version == InvertedIndexStorageFormatPB::V1) { + auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id); + if (inverted_index_info.index_info().empty()) { + VLOG_DEBUG << "No index info available for segment " << segment_id; + continue; + } + for (const auto& info : inverted_index_info.index_info()) { + g_file_cache_event_driven_warm_up_requested_index_num << 1; + if (info.index_file_size() != -1) { + g_file_cache_event_driven_warm_up_requested_index_size + << info.index_file_size(); + } else { + VLOG_DEBUG << "Invalid index_file_size for segment_id " << segment_id + << ", index_id " << info.index_id(); + } + } + } else { // InvertedIndexStorageFormatPB::V2 + auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id); + g_file_cache_event_driven_warm_up_requested_index_num << 1; + if (inverted_index_info.has_index_size()) { + g_file_cache_event_driven_warm_up_requested_index_size + << inverted_index_info.index_size(); + } else { + VLOG_DEBUG << "index_size is not set for segment " << segment_id; + } + } + } + } + + brpc::Controller cntl; + PWarmUpRowsetResponse response; + brpc_stub->warm_up_rowset(&cntl, &request, &response, nullptr); + } +} + +void CloudWarmUpManager::recycle_cache( + int64_t tablet_id, const std::vector& rowset_ids, + const std::vector& num_segments, + const std::vector>& index_file_names) { + LOG(INFO) << "recycle_cache: tablet_id=" << tablet_id << ", num_rowsets=" << rowset_ids.size(); + auto replicas = get_replica_info(tablet_id); + if (replicas.empty()) { + return; + } + if (rowset_ids.size() != num_segments.size()) { + LOG(WARNING) << "recycle_cache: rowset_ids size mismatch with num_segments"; + return; + } + + PRecycleCacheRequest request; + for (int i = 0; i < rowset_ids.size(); i++) { + RecycleCacheMeta* meta = request.add_cache_metas(); + meta->set_tablet_id(tablet_id); + meta->set_rowset_id(rowset_ids[i].to_string()); + meta->set_num_segments(num_segments[i]); + for (const auto& name : index_file_names[i]) { + meta->add_index_file_names(name); + } + g_file_cache_recycle_cache_requested_segment_num << num_segments[i]; + g_file_cache_recycle_cache_requested_index_num << index_file_names[i].size(); + } + for (auto& replica : replicas) { + // send sync request + std::string host = replica.host; + auto dns_cache = ExecEnv::GetInstance()->dns_cache(); + if (dns_cache == nullptr) { + LOG(WARNING) << "DNS cache is not initialized, skipping hostname resolve"; + } else if (!is_valid_ip(replica.host)) { + Status status = dns_cache->get(replica.host, &host); + if (!status.ok()) { + LOG(WARNING) << "failed to get ip from host " << replica.host << ": " + << status.to_string(); + return; + } + } + std::string brpc_addr = get_host_port(host, replica.brpc_port); + Status st = Status::OK(); + std::shared_ptr brpc_stub = + ExecEnv::GetInstance()->brpc_internal_client_cache()->get_new_client_no_cache( + brpc_addr); + if (!brpc_stub) { + st = Status::RpcError("Address {} is wrong", brpc_addr); + continue; + } + brpc::Controller cntl; + PRecycleCacheResponse response; + brpc_stub->recycle_cache(&cntl, &request, &response, nullptr); + } +} + } // namespace doris diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index 219dedc58065a6..55fbcc476da25d 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -67,8 +67,17 @@ class CloudWarmUpManager { // Cancel the job Status clear_job(int64_t job_id); + Status set_event(int64_t job_id, TWarmUpEventType::type event, bool clear = false); + + void warm_up_rowset(RowsetMeta& rs_meta); + + void recycle_cache(int64_t tablet_id, const std::vector& rowset_ids, + const std::vector& num_segments, + const std::vector>& index_file_names); + private: void handle_jobs(); + std::vector get_replica_info(int64_t tablet_id); std::mutex _mtx; std::condition_variable _cond; @@ -80,6 +89,13 @@ class CloudWarmUpManager { bool _closed {false}; // the attribute for compile in ut [[maybe_unused]] CloudStorageEngine& _engine; + + // timestamp, info + using CacheEntry = std::pair; + // tablet_id -> entry + using Cache = std::unordered_map; + // job_id -> cache + std::unordered_map _tablet_replica_cache; }; } // namespace doris diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index 7198ed128f6666..304eeae43f703b 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -90,5 +90,9 @@ DEFINE_mInt32(meta_service_conflict_error_retry_times, "10"); DEFINE_Bool(enable_check_storage_vault, "true"); +DEFINE_mInt64(warmup_tablet_replica_info_cache_ttl_sec, "600"); + +DEFINE_mInt64(warm_up_rowset_slow_log_ms, "1000"); + #include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index e1f9e0de63b487..5e3e04a744bf1f 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -126,5 +126,9 @@ DECLARE_mInt32(meta_service_conflict_error_retry_times); DECLARE_Bool(enable_check_storage_vault); +DECLARE_mInt64(warmup_tablet_replica_info_cache_ttl_sec); + +DECLARE_mInt64(warm_up_rowset_slow_log_ms); + #include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/common/config.h b/be/src/common/config.h index 55ab5230be7001..037353fe0a4008 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1168,6 +1168,8 @@ DECLARE_mInt64(file_cache_background_monitor_interval_ms); DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms); DECLARE_mInt64(file_cache_background_ttl_gc_batch); +DECLARE_mBool(enable_reader_dryrun_when_download_file_cache); + // inverted index searcher cache // cache entry stay time after lookup DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s); diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index b9944e39989d2b..dc4c622e807daf 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -40,6 +40,12 @@ namespace doris::io { +bvar::Adder g_file_cache_download_submitted_size("file_cache_download_submitted_size"); +bvar::Adder g_file_cache_download_finished_size("file_cache_download_finished_size"); +bvar::Adder g_file_cache_download_submitted_num("file_cache_download_submitted_num"); +bvar::Adder g_file_cache_download_finished_num("file_cache_download_finished_num"); +bvar::Adder g_file_cache_download_failed_num("file_cache_download_failed_num"); + FileCacheBlockDownloader::FileCacheBlockDownloader(CloudStorageEngine& engine) : _engine(engine) { _poller = std::thread(&FileCacheBlockDownloader::polling_download_task, this); auto st = ThreadPoolBuilder("FileCacheBlockDownloader") @@ -75,6 +81,14 @@ void FileCacheBlockDownloader::submit_download_task(DownloadTask task) { std::lock_guard lock(_inflight_mtx); for (auto& meta : std::get<0>(task.task_message)) { ++_inflight_tablets[meta.tablet_id()]; + if (meta.size() > 0) { + g_file_cache_download_submitted_size << meta.size(); + } + } + } else { + int64_t download_size = std::get<1>(task.task_message).download_size; + if (download_size > 0) { + g_file_cache_download_submitted_size << download_size; } } @@ -87,12 +101,14 @@ void FileCacheBlockDownloader::submit_download_task(DownloadTask task) { download_file_meta.download_done( Status::InternalError("The downloader queue is full")); } + g_file_cache_download_failed_num << 1; } _task_queue.pop_front(); // Eliminate the earliest task in the queue } _task_queue.push_back(std::move(task)); _empty.notify_all(); } + g_file_cache_download_submitted_num << 1; } void FileCacheBlockDownloader::polling_download_task() { @@ -212,6 +228,7 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met if (meta.download_done) { meta.download_done(std::move(st)); } + g_file_cache_download_failed_num << 1; return; } @@ -237,13 +254,16 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met if (meta.download_done) { meta.download_done(std::move(st)); } + g_file_cache_download_failed_num << 1; return; } + g_file_cache_download_finished_size << size; } if (meta.download_done) { meta.download_done(Status::OK()); } + g_file_cache_download_finished_num << 1; } void FileCacheBlockDownloader::download_blocks(DownloadTask& task) { diff --git a/be/src/olap/rowset/rowset_meta.cpp b/be/src/olap/rowset/rowset_meta.cpp index c9851cdc5fc64b..d3e67c3fad4bf9 100644 --- a/be/src/olap/rowset/rowset_meta.cpp +++ b/be/src/olap/rowset/rowset_meta.cpp @@ -210,7 +210,7 @@ void RowsetMeta::add_segments_file_size(const std::vector& seg_file_size } } -int64_t RowsetMeta::segment_file_size(int seg_id) { +int64_t RowsetMeta::segment_file_size(int seg_id) const { DCHECK(_rowset_meta_pb.segments_file_size().empty() || _rowset_meta_pb.segments_file_size_size() > seg_id) << _rowset_meta_pb.segments_file_size_size() << ' ' << seg_id; diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index 4421b6dda1fb4e..2a4650611354c3 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -356,7 +356,7 @@ class RowsetMeta : public MetadataAdder { void add_segments_file_size(const std::vector& seg_file_size); // Return -1 if segment file size is unknown - int64_t segment_file_size(int seg_id); + int64_t segment_file_size(int seg_id) const; const auto& segments_file_size() const { return _rowset_meta_pb.segments_file_size(); } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 23da54f52a0281..fe4f2fbba5b118 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3233,6 +3233,12 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true, masterOnly = true) public static int cloud_warm_up_job_scheduler_interval_millisecond = 1000; // 1 seconds + @ConfField(mutable = true, masterOnly = true) + public static long cloud_warm_up_job_max_bytes_per_batch = 21474836480L; // 20GB + + @ConfField(mutable = true, masterOnly = true) + public static boolean cloud_warm_up_force_all_partitions = false; + @ConfField(mutable = true, masterOnly = true) public static boolean enable_fetch_cluster_cache_hotspot = true; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index efaebb8d445702..bd4d91bcbdd680 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -227,7 +227,8 @@ unsupportedOtherStatement | UNLOCK TABLES #unlockTables | WARM UP (CLUSTER | COMPUTE GROUP) destination=identifier WITH ((CLUSTER | COMPUTE GROUP) source=identifier | - (warmUpItem (AND warmUpItem)*)) FORCE? #warmUpCluster + (warmUpItem (AND warmUpItem)*)) FORCE? + properties=propertyClause? #warmUpCluster | BACKUP SNAPSHOT label=multipartIdentifier TO repo=identifier ((ON | EXCLUDE) LEFT_PAREN baseTableRef (COMMA baseTableRef)* RIGHT_PAREN)? properties=propertyClause? #backup diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 919583b706784c..4f3a4dbbb3cfc9 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -1473,17 +1473,17 @@ alter_stmt ::= ; warm_up_stmt ::= - KW_WARM KW_UP KW_CLUSTER ident:dstClusterName KW_WITH KW_CLUSTER ident:srcClusterName opt_force:force + KW_WARM KW_UP KW_CLUSTER ident:dstClusterName KW_WITH KW_CLUSTER ident:srcClusterName opt_force:force opt_properties:properties {: - RESULT = new WarmUpClusterStmt(dstClusterName, srcClusterName, force); + RESULT = new WarmUpClusterStmt(dstClusterName, srcClusterName, force, properties); :} | KW_WARM KW_UP KW_CLUSTER ident:dstClusterName KW_WITH warm_up_list:list opt_force:force {: RESULT = new WarmUpClusterStmt(dstClusterName, list, force); :} - | KW_WARM KW_UP KW_COMPUTE KW_GROUP ident:dstClusterName KW_WITH KW_COMPUTE KW_GROUP ident:srcClusterName opt_force:force + | KW_WARM KW_UP KW_COMPUTE KW_GROUP ident:dstClusterName KW_WITH KW_COMPUTE KW_GROUP ident:srcClusterName opt_force:force opt_properties:properties {: - RESULT = new WarmUpClusterStmt(dstClusterName, srcClusterName, force); + RESULT = new WarmUpClusterStmt(dstClusterName, srcClusterName, force, properties); :} | KW_WARM KW_UP KW_COMPUTE KW_GROUP ident:dstClusterName KW_WITH warm_up_list:list opt_force:force {: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java index 9ec063d3f76164..725025e77b8789 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java @@ -36,10 +36,13 @@ public class ShowCloudWarmUpStmt extends ShowStmt implements NotFallbackInParser private static final ImmutableList WARM_UP_JOB_TITLE_NAMES = new ImmutableList.Builder() .add("JobId") - .add("ComputeGroup") + .add("SrcComputeGroup") + .add("DstComputeGroup") .add("Status") .add("Type") + .add("SyncMode") .add("CreateTime") + .add("StartTime") .add("FinishBatch") .add("AllBatch") .add("FinishTime") diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java index cca21d5c259c06..04d66eb8309281 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -46,12 +47,15 @@ public class WarmUpClusterStmt extends StatementBase implements NotFallbackInPar private String srcClusterName; private boolean isWarmUpWithTable; private boolean isForce; + private Map properties; - public WarmUpClusterStmt(String dstClusterName, String srcClusterName, boolean isForce) { + public WarmUpClusterStmt(String dstClusterName, String srcClusterName, boolean isForce, + Map properties) { this.dstClusterName = dstClusterName; this.srcClusterName = srcClusterName; this.isForce = isForce; this.isWarmUpWithTable = false; + this.properties = properties; } public WarmUpClusterStmt(String dstClusterName, List> tableList, boolean isForce) { @@ -59,6 +63,7 @@ public WarmUpClusterStmt(String dstClusterName, List> tab this.tableList = tableList; this.isForce = isForce; this.isWarmUpWithTable = true; + this.properties = new HashMap<>(); } @Override @@ -161,4 +166,8 @@ public boolean isWarmUpWithTable() { public boolean isForce() { return isForce; } + + public Map getProperties() { + return properties; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java index b73e467836d91c..167e413a02e996 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java @@ -29,6 +29,7 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.cloud.CloudWarmUpJob.JobState; import org.apache.doris.cloud.CloudWarmUpJob.JobType; +import org.apache.doris.cloud.CloudWarmUpJob.SyncMode; import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.AnalysisException; @@ -70,6 +71,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -107,11 +109,137 @@ public class CacheHotspotManager extends MasterDaemon { private ConcurrentMap runnableCloudWarmUpJobs = Maps.newConcurrentMap(); - private Set runnableClusterSet = ConcurrentHashMap.newKeySet(); - private final ThreadPoolExecutor cloudWarmUpThreadPool = ThreadPoolManager.newDaemonCacheThreadPool( Config.max_active_cloud_warm_up_job, "cloud-warm-up-pool", true); + private static class JobKey { + private final String srcName; + private final String dstName; + private final CloudWarmUpJob.SyncMode syncMode; + + public JobKey(String srcName, String dstName, CloudWarmUpJob.SyncMode syncMode) { + this.srcName = srcName; + this.dstName = dstName; + this.syncMode = syncMode; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof JobKey)) { + return false; + } + JobKey jobKey = (JobKey) o; + return Objects.equals(srcName, jobKey.srcName) + && Objects.equals(dstName, jobKey.dstName) + && syncMode == jobKey.syncMode; + } + + @Override + public int hashCode() { + return Objects.hash(srcName, dstName, syncMode); + } + + @Override + public String toString() { + return "WarmUpJob src='" + srcName + "', dst='" + dstName + "', syncMode=" + String.valueOf(syncMode); + } + } + + // Tracks long-running jobs (event-driven and periodic). + // Ensures only one active job exists per tuple. + private Set repeatJobDetectionSet = ConcurrentHashMap.newKeySet(); + + private void registerJobForRepeatDetection(CloudWarmUpJob job, boolean replay) throws AnalysisException { + if (job.isDone()) { + return; + } + if (job.isEventDriven() || job.isPeriodic()) { + // For long lasting jobs, i.e. event-driven and periodic. + // It is meaningless to create more than one job for a given src, dst, and syncMode. + JobKey key = new JobKey(job.getSrcClusterName(), job.getDstClusterName(), job.getSyncMode()); + boolean added = this.repeatJobDetectionSet.add(key); + if (!added && !replay) { + throw new AnalysisException(key + " already has a runnable job"); + } + } + } + + // Tracks warm-up jobs scheduled by CacheHotSpotManager. + // Ensures that at most one job runs concurrently per destination cluster. + private Map clusterToRunningJobId = new ConcurrentHashMap<>(); + + /** + * Attempts to register a job as running for the given destination cluster. + *

+ * For one-time or periodic jobs, returns {@code false} if there is already a running job + * for the specified destination cluster. Returns {@code true} if this job is successfully + * registered as the only running job for that cluster. + *

+ * For event-driven jobs, this method does not perform any registration and always returns {@code true}. + * + * @param job the CloudWarmUpJob to register + * @return {@code true} if the job was registered successfully or is event-driven; {@code false} otherwise + */ + public boolean tryRegisterRunningJob(CloudWarmUpJob job) { + if (job.isEventDriven()) { + // Event-driven jobs do not require registration, always allow + return true; + } + + String clusterName = job.getDstClusterName(); + long jobId = job.getJobId(); + + // Try to register the job atomically if absent + Long existingJobId = clusterToRunningJobId.putIfAbsent(clusterName, jobId); + boolean success = (existingJobId == null) || (existingJobId == jobId); + if (!success) { + LOG.info("Job {} skipped: waiting for job {} to finish on destination cluster {}", + jobId, existingJobId, clusterName); + } + return success; + } + + /** + * Deregisters the given job from the running jobs map, allowing another job + * to run on the same destination cluster. + *

+ * For event-driven jobs, this method does nothing and always returns {@code true} + * since they are not registered. + *

+ * This method only removes the job if the currently registered job ID matches + * the job's ID, ensuring no accidental deregistration of other jobs. + * + * @param job the CloudWarmUpJob to deregister + * @return {@code true} if the job was successfully deregistered or is event-driven; {@code false} otherwise + */ + private boolean deregisterRunningJob(CloudWarmUpJob job) { + if (job.isEventDriven()) { + // Event-driven jobs are not registered, so nothing to deregister + return true; + } + + String clusterName = job.getDstClusterName(); + long jobId = job.getJobId(); + + return clusterToRunningJobId.remove(clusterName, jobId); + } + + public void notifyJobStop(CloudWarmUpJob job) { + if (job.isOnce() || job.isPeriodic()) { + this.deregisterRunningJob(job); + } + if (!job.isDone()) { + return; + } + if (job.isEventDriven() || job.isPeriodic()) { + this.repeatJobDetectionSet.remove(new JobKey( + job.getSrcClusterName(), job.getDstClusterName(), job.getSyncMode())); + } + } + public CacheHotspotManager(CloudSystemInfoService nodeMgr) { super("CacheHotspotManager", Config.fetch_cluster_cache_hotspot_interval_ms); this.nodeMgr = nodeMgr; @@ -364,13 +492,13 @@ Long getFileCacheCapacity(String clusterName) throws RuntimeException { return totalFileCache; } - private Map>> splitBatch(Map> beToWarmUpTablets) { - final Long maxSizePerBatch = 10737418240L; // 10G + public Map>> splitBatch(Map> beToWarmUpTablets) { + final Long maxSizePerBatch = Config.cloud_warm_up_job_max_bytes_per_batch; Map>> beToTabletIdBatches = new HashMap<>(); for (Map.Entry> entry : beToWarmUpTablets.entrySet()) { List> batches = new ArrayList<>(); List batch = new ArrayList<>(); - Long curBatchSize = 0L; + long curBatchSize = 0L; for (Tablet tablet : entry.getValue()) { if (curBatchSize + tablet.getDataSize(true) > maxSizePerBatch) { batches.add(batch); @@ -388,7 +516,7 @@ private Map>> splitBatch(Map> beToWarmU return beToTabletIdBatches; } - private Map> warmUpNewClusterByCluster(String dstClusterName, String srcClusterName) { + private List getHotTablets(String srcClusterName, String dstClusterName) { Long dstTotalFileCache = getFileCacheCapacity(dstClusterName); List> result = getClusterTopNHotPartitions(srcClusterName); Long warmUpTabletsSize = 0L; @@ -427,6 +555,39 @@ private Map> warmUpNewClusterByCluster(String dstClusterName, } } Collections.reverse(tablets); + return tablets; + } + + private List getAllTablets(String srcClusterName, String dstClusterName) { + List tablets = new ArrayList<>(); + List dbs = Env.getCurrentInternalCatalog().getDbs(); + for (Database db : dbs) { + List tables = db.getTables(); + for (Table table : tables) { + if (!(table instanceof OlapTable)) { + continue; + } + OlapTable olapTable = (OlapTable) table; + for (Partition partition : olapTable.getPartitions()) { + // Maybe IndexExtState.ALL + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (Tablet tablet : index.getTablets()) { + tablets.add(tablet); + } + } + } + } + } + return tablets; + } + + public Map> warmUpNewClusterByCluster(String dstClusterName, String srcClusterName) { + List tablets; + if (Config.cloud_warm_up_force_all_partitions) { + tablets = getAllTablets(srcClusterName, dstClusterName); + } else { + tablets = getHotTablets(srcClusterName, dstClusterName); + } List backends = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) .getBackendsByClusterName(dstClusterName); Map> beToWarmUpTablets = new HashMap<>(); @@ -497,8 +658,8 @@ public Map getCloudWarmUpJobs() { return this.cloudWarmUpJobs; } - public Set getRunnableClusterSet() { - return this.runnableClusterSet; + public CloudWarmUpJob getCloudWarmUpJob(long jobId) { + return this.cloudWarmUpJobs.get(jobId); } public List> getAllJobInfos(int limit) { @@ -511,13 +672,11 @@ public List> getAllJobInfos(int limit) { return infos; } - public void addCloudWarmUpJob(CloudWarmUpJob job) { + public void addCloudWarmUpJob(CloudWarmUpJob job) throws AnalysisException { + registerJobForRepeatDetection(job, false); cloudWarmUpJobs.put(job.getJobId(), job); LOG.info("add cloud warm up job {}", job.getJobId()); runnableCloudWarmUpJobs.put(job.getJobId(), job); - if (!job.isDone()) { - runnableClusterSet.add(job.getCloudClusterName()); - } } public List getPartitionsFromTriple(Triple tableTriple) { @@ -616,51 +775,84 @@ public Map> warmUpNewClusterByTable(long jobId, String dstClu } public long createJob(WarmUpClusterStmt stmt) throws AnalysisException { - if (runnableClusterSet.contains(stmt.getDstClusterName())) { - throw new AnalysisException("cluster: " + stmt.getDstClusterName() + " already has a runnable job"); - } - Map> beToWarmUpTablets = new HashMap<>(); long jobId = Env.getCurrentEnv().getNextId(); - if (!FeConstants.runningUnitTest) { - if (stmt.isWarmUpWithTable()) { + CloudWarmUpJob warmUpJob; + if (stmt.isWarmUpWithTable()) { + Map> beToWarmUpTablets = new HashMap<>(); + if (!FeConstants.runningUnitTest) { beToWarmUpTablets = warmUpNewClusterByTable(jobId, stmt.getDstClusterName(), stmt.getTables(), stmt.isForce()); + } + Map>> beToTabletIdBatches = splitBatch(beToWarmUpTablets); + warmUpJob = new CloudWarmUpJob(jobId, null, stmt.getDstClusterName(), + beToTabletIdBatches, JobType.TABLE); + } else { + CloudWarmUpJob.Builder builder = new CloudWarmUpJob.Builder() + .setJobId(jobId) + .setSrcClusterName(stmt.getSrcClusterName()) + .setDstClusterName(stmt.getDstClusterName()) + .setJobType(JobType.CLUSTER); + + Map properties = stmt.getProperties(); + if ("periodic".equals(properties.get("sync_mode"))) { + String syncIntervalSecStr = properties.get("sync_interval_sec"); + if (syncIntervalSecStr == null) { + throw new AnalysisException("No sync_interval_sec is provided"); + } + long syncIntervalSec; + try { + syncIntervalSec = Long.parseLong(syncIntervalSecStr); + } catch (NumberFormatException e) { + throw new AnalysisException("Illegal sync_interval_sec: " + syncIntervalSecStr); + } + builder.setSyncMode(SyncMode.PERIODIC) + .setSyncInterval(syncIntervalSec); + } else if ("event_driven".equals(properties.get("sync_mode"))) { + String syncEventStr = properties.get("sync_event"); + if (syncEventStr == null) { + throw new AnalysisException("No sync_event is provided"); + } + CloudWarmUpJob.SyncEvent syncEvent; + try { + syncEvent = CloudWarmUpJob.SyncEvent.valueOf(syncEventStr.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new AnalysisException("Illegal sync_event: " + syncEventStr, e); + } + builder.setSyncMode(SyncMode.EVENT_DRIVEN) + .setSyncEvent(syncEvent); } else { - beToWarmUpTablets = warmUpNewClusterByCluster(stmt.getDstClusterName(), stmt.getSrcClusterName()); + builder.setSyncMode(SyncMode.ONCE); } + warmUpJob = builder.build(); } - Map>> beToTabletIdBatches = splitBatch(beToWarmUpTablets); - - CloudWarmUpJob.JobType jobType = stmt.isWarmUpWithTable() ? JobType.TABLE : JobType.CLUSTER; - CloudWarmUpJob warmUpJob; - if (jobType == JobType.TABLE) { - warmUpJob = new CloudWarmUpJob(jobId, stmt.getDstClusterName(), beToTabletIdBatches, jobType, - stmt.getTables(), stmt.isForce()); - } else { - warmUpJob = new CloudWarmUpJob(jobId, stmt.getDstClusterName(), beToTabletIdBatches, jobType); - } addCloudWarmUpJob(warmUpJob); Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(warmUpJob); LOG.info("finished to create cloud warm up job: {}", warmUpJob.getJobId()); return jobId; - } public void cancel(CancelCloudWarmUpStmt stmt) throws DdlException { - CloudWarmUpJob job = cloudWarmUpJobs.get(stmt.getJobId()); + cancel(stmt.getJobId()); + } + + public void cancel(long jobId) throws DdlException { + CloudWarmUpJob job = cloudWarmUpJobs.get(jobId); if (job == null) { - throw new DdlException("job id: " + stmt.getJobId() + " does not exist."); + throw new DdlException("job id: " + jobId + " does not exist."); } - if (!job.cancel("user cancel")) { + if (!job.cancel("user cancel", true)) { throw new DdlException("job can not be cancelled. State: " + job.getJobState()); } } private void runCloudWarmUpJob() { runnableCloudWarmUpJobs.values().forEach(cloudWarmUpJob -> { + if (cloudWarmUpJob.shouldWait()) { + return; + } if (!cloudWarmUpJob.isDone() && !activeCloudWarmUpJobs.containsKey(cloudWarmUpJob.getJobId()) && activeCloudWarmUpJobs.size() < Config.max_active_cloud_warm_up_job) { if (FeConstants.runningUnitTest) { @@ -686,9 +878,9 @@ public void replayCloudWarmUpJob(CloudWarmUpJob cloudWarmUpJob) throws Exception cloudWarmUpJobs.put(cloudWarmUpJob.getJobId(), cloudWarmUpJob); LOG.info("replay cloud warm up job {}, state {}", cloudWarmUpJob.getJobId(), cloudWarmUpJob.getJobState()); if (cloudWarmUpJob.isDone()) { - runnableClusterSet.remove(cloudWarmUpJob.getCloudClusterName()); + notifyJobStop(cloudWarmUpJob); } else { - runnableClusterSet.add(cloudWarmUpJob.getCloudClusterName()); + registerJobForRepeatDetection(cloudWarmUpJob, true); } if (cloudWarmUpJob.jobState == JobState.DELETED) { if (cloudWarmUpJobs.remove(cloudWarmUpJob.getJobId()) != null diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java index 463a37c4635dc5..0bd655893a7a7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java @@ -18,6 +18,7 @@ package org.apache.doris.cloud; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Tablet; import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.ClientPool; @@ -27,7 +28,9 @@ import org.apache.doris.common.Triple; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.system.Backend; @@ -36,6 +39,7 @@ import org.apache.doris.thrift.TJobMeta; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TWarmUpEventType; import org.apache.doris.thrift.TWarmUpTabletsRequest; import org.apache.doris.thrift.TWarmUpTabletsRequestType; import org.apache.doris.thrift.TWarmUpTabletsResponse; @@ -75,26 +79,43 @@ public enum JobType { TABLE; } + public enum SyncMode { + ONCE, + PERIODIC, + EVENT_DRIVEN; + } + + public enum SyncEvent { + LOAD, + QUERY + } + @SerializedName(value = "jobId") protected long jobId; @SerializedName(value = "jobState") protected JobState jobState; @SerializedName(value = "createTimeMs") protected long createTimeMs = -1; + @SerializedName(value = "startTimeMs") + protected long startTimeMs = -1; @SerializedName(value = "errMsg") protected String errMsg = ""; @SerializedName(value = "finishedTimeMs") protected long finishedTimeMs = -1; + @SerializedName(value = "srcClusterName") + protected String srcClusterName = ""; + + // the serialized name is kept for compatibility reasons @SerializedName(value = "cloudClusterName") - protected String cloudClusterName = ""; + protected String dstClusterName = ""; @SerializedName(value = "lastBatchId") protected long lastBatchId = -1; @SerializedName(value = "beToTabletIdBatches") - protected Map>> beToTabletIdBatches; + protected Map>> beToTabletIdBatches = new HashMap<>(); @SerializedName(value = "beToThriftAddress") protected Map beToThriftAddress = new HashMap<>(); @@ -108,6 +129,15 @@ public enum JobType { @SerializedName(value = "force") protected boolean force = false; + @SerializedName(value = "syncMode") + protected SyncMode syncMode = SyncMode.ONCE; + + @SerializedName(value = "syncInterval") + protected long syncInterval; + + @SerializedName(value = "syncEvent") + protected SyncEvent syncEvent; + private Map beToClient; private Map beToAddr; @@ -120,17 +150,93 @@ public enum JobType { private boolean setJobDone = false; - public CloudWarmUpJob(long jobId, String cloudClusterName, - Map>> beToTabletIdBatches, JobType jobType) { + public static class Builder { + private long jobId; + private String srcClusterName; + private String dstClusterName; + private JobType jobType = JobType.CLUSTER; + private SyncMode syncMode = SyncMode.ONCE; + private SyncEvent syncEvent; + private long syncInterval; + + public Builder() {} + + public Builder setJobId(long jobId) { + this.jobId = jobId; + return this; + } + + public Builder setSrcClusterName(String srcClusterName) { + this.srcClusterName = srcClusterName; + return this; + } + + public Builder setDstClusterName(String dstClusterName) { + this.dstClusterName = dstClusterName; + return this; + } + + public Builder setJobType(JobType jobType) { + this.jobType = jobType; + return this; + } + + public Builder setSyncMode(SyncMode syncMode) { + this.syncMode = syncMode; + return this; + } + + public Builder setSyncEvent(SyncEvent syncEvent) { + this.syncEvent = syncEvent; + return this; + } + + public Builder setSyncInterval(long syncInterval) { + this.syncInterval = syncInterval; + return this; + } + + public CloudWarmUpJob build() { + if (jobId == 0 || srcClusterName == null || dstClusterName == null || jobType == null || syncMode == null) { + throw new IllegalStateException("Missing required fields for CloudWarmUpJob"); + } + return new CloudWarmUpJob(this); + } + } + + private CloudWarmUpJob(Builder builder) { + this.jobId = builder.jobId; + this.jobState = JobState.PENDING; + this.srcClusterName = builder.srcClusterName; + this.dstClusterName = builder.dstClusterName; + this.jobType = builder.jobType; + this.syncMode = builder.syncMode; + this.syncEvent = builder.syncEvent; + this.syncInterval = builder.syncInterval; + this.createTimeMs = System.currentTimeMillis(); + } + + private void fetchBeToThriftAddress() { + String clusterName = isEventDriven() ? srcClusterName : dstClusterName; + List backends = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getBackendsByClusterName(clusterName); + for (Backend backend : backends) { + beToThriftAddress.put(backend.getId(), backend.getHost() + ":" + backend.getBePort()); + } + } + + public CloudWarmUpJob(long jobId, String srcClusterName, String dstClusterName, + Map>> beToTabletIdBatches, JobType jobType) { this.jobId = jobId; this.jobState = JobState.PENDING; - this.cloudClusterName = cloudClusterName; + this.srcClusterName = srcClusterName; + this.dstClusterName = dstClusterName; this.beToTabletIdBatches = beToTabletIdBatches; this.createTimeMs = System.currentTimeMillis(); this.jobType = jobType; if (!FeConstants.runningUnitTest) { List backends = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getBackendsByClusterName(cloudClusterName); + .getBackendsByClusterName(dstClusterName); for (Backend backend : backends) { beToThriftAddress.put(backend.getId(), backend.getHost() + ":" + backend.getBePort()); } @@ -140,11 +246,68 @@ public CloudWarmUpJob(long jobId, String cloudClusterName, public CloudWarmUpJob(long jobId, String cloudClusterName, Map>> beToTabletIdBatches, JobType jobType, List> tables, boolean force) { - this(jobId, cloudClusterName, beToTabletIdBatches, jobType); + this(jobId, null, cloudClusterName, beToTabletIdBatches, jobType); this.tables = tables; this.force = force; } + public void fetchBeToTabletIdBatches() { + if (FeConstants.runningUnitTest) { + return; + } + if (jobType == JobType.TABLE) { + // warm up with table will have to set tablets on creation + return; + } + if (syncMode == null) { + // This job was created by an old FE version. + // It doesn't have the source cluster name, but tablets were already set. + // Return for backward compatibility. + return; + } + if (this.isEventDriven()) { + // Event-driven jobs do not need to calculate tablets + return; + } + CacheHotspotManager manager = ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr(); + Map> beToWarmUpTablets = + manager.warmUpNewClusterByCluster(dstClusterName, srcClusterName); + long totalTablets = beToWarmUpTablets.values().stream() + .mapToLong(List::size) + .sum(); + beToTabletIdBatches = manager.splitBatch(beToWarmUpTablets); + long totalBatches = beToTabletIdBatches.values().stream() + .mapToLong(List::size) + .sum(); + LOG.info("warm up job {} tablet num {}, batch num {}", jobId, totalTablets, totalBatches); + } + + public boolean shouldWait() { + if (!this.isPeriodic()) { + return false; + } + if (this.jobState != JobState.PENDING) { + return false; + } + long timeSinceLastStart = System.currentTimeMillis() - this.startTimeMs; + if (timeSinceLastStart < this.syncInterval * 1000L) { + return true; + } + return false; + } + + public boolean isOnce() { + return this.syncMode == SyncMode.ONCE || this.syncMode == null; + } + + public boolean isPeriodic() { + return this.syncMode == SyncMode.PERIODIC; + } + + public boolean isEventDriven() { + return this.syncMode == SyncMode.EVENT_DRIVEN; + } + public long getJobId() { return jobId; } @@ -181,20 +344,51 @@ public JobType getJobType() { return jobType; } + public SyncMode getSyncMode() { + return syncMode; + } + + public String getSyncModeString() { + if (syncMode == null) { + // For backward compatibility: older FE versions did not set syncMode for jobs, + // so default to ONCE when syncMode is missing. + return String.valueOf(SyncMode.ONCE); + } + StringBuilder sb = new StringBuilder().append(syncMode); + switch (syncMode) { + case PERIODIC: + sb.append(" ("); + sb.append(syncInterval); + sb.append("s)"); + break; + case EVENT_DRIVEN: + sb.append(" ("); + sb.append(syncEvent); + sb.append(")"); + break; + default: + break; + } + return sb.toString(); + } + public List getJobInfo() { List info = Lists.newArrayList(); info.add(String.valueOf(jobId)); - info.add(cloudClusterName); - info.add(jobState.name()); - info.add(jobType.name()); + info.add(srcClusterName); + info.add(dstClusterName); + info.add(String.valueOf(jobState)); + info.add(String.valueOf(jobType)); + info.add(this.getSyncModeString()); info.add(TimeUtils.longToTimeStringWithms(createTimeMs)); + info.add(TimeUtils.longToTimeStringWithms(startTimeMs)); info.add(Long.toString(lastBatchId + 1)); long maxBatchSize = 0; - for (List> list : beToTabletIdBatches.values()) { - long size = list.size(); - if (size > maxBatchSize) { - maxBatchSize = size; - } + if (beToTabletIdBatches != null) { + maxBatchSize = beToTabletIdBatches.values().stream() + .mapToLong(List::size) + .max() + .orElse(0); } info.add(Long.toString(maxBatchSize)); info.add(TimeUtils.longToTimeStringWithms(finishedTimeMs)); @@ -224,7 +418,7 @@ public void setFinishedTimeMs(long timeMs) { } public void setCloudClusterName(String name) { - this.cloudClusterName = name; + this.dstClusterName = name; } public void setLastBatchId(long id) { @@ -248,7 +442,8 @@ public boolean isDone() { } public boolean isTimeout() { - return (System.currentTimeMillis() - createTimeMs) / 1000 > Config.cloud_warm_up_timeout_second; + return jobState == JobState.RUNNING + && (System.currentTimeMillis() - startTimeMs) / 1000 > Config.cloud_warm_up_timeout_second; } public boolean isExpire() { @@ -256,20 +451,24 @@ public boolean isExpire() { > Config.history_cloud_warm_up_job_keep_max_second; } - public String getCloudClusterName() { - return cloudClusterName; + public String getDstClusterName() { + return dstClusterName; + } + + public String getSrcClusterName() { + return srcClusterName; } public synchronized void run() { if (isTimeout()) { - cancel("Timeout"); + cancel("Timeout", false); return; } if (Config.isCloudMode()) { LOG.debug("set context to job"); ConnectContext ctx = new ConnectContext(); ctx.setThreadLocalInfo(); - ctx.setCloudCluster(cloudClusterName); + ctx.setCloudCluster(dstClusterName); } try { switch (jobState) { @@ -294,6 +493,9 @@ public synchronized void run() { } public void initClients() throws Exception { + if (beToThriftAddress == null || beToThriftAddress.isEmpty()) { + fetchBeToThriftAddress(); + } if (beToClient == null) { beToClient = new HashMap<>(); beToAddr = new HashMap<>(); @@ -333,39 +535,77 @@ public void releaseClients() { beToAddr = null; } - public final synchronized boolean cancel(String errMsg) { - if (this.jobState.isFinalState()) { - return false; - } + private final void clearJobOnBEs() { try { initClients(); for (Map.Entry entry : beToClient.entrySet()) { TWarmUpTabletsRequest request = new TWarmUpTabletsRequest(); request.setType(TWarmUpTabletsRequestType.CLEAR_JOB); request.setJobId(jobId); - LOG.info("send warm up request. request_type=CLEAR_JOB"); + if (this.isEventDriven()) { + TWarmUpEventType event = getTWarmUpEventType(); + if (event == null) { + throw new IllegalArgumentException("Unknown SyncEvent " + syncEvent); + } + request.setEvent(event); + } + LOG.info("send warm up request to BE {}. job_id={}, request_type=CLEAR_JOB", + entry.getKey(), jobId); entry.getValue().warmUpTablets(request); } } catch (Exception e) { - LOG.warn("warm up job {} cancel exception: {}", jobId, e.getMessage()); + LOG.warn("send warm up request failed. job_id={}, request_type=CLEAR_JOB, exception={}", + jobId, e.getMessage()); } finally { releaseClients(); } - this.jobState = JobState.CANCELLED; + } + + public final synchronized boolean cancel(String errMsg, boolean force) { + if (this.jobState.isFinalState()) { + return false; + } + if (this.jobState == JobState.PENDING) { + // BE haven't started this job yet, skip RPC + } else { + clearJobOnBEs(); + } + if (this.isOnce() || force) { + this.jobState = JobState.CANCELLED; + } else { + this.jobState = JobState.PENDING; + } this.errMsg = errMsg; this.finishedTimeMs = System.currentTimeMillis(); + MetricRepo.updateClusterWarmUpJobLastFinishTime(String.valueOf(jobId), srcClusterName, + dstClusterName, finishedTimeMs); LOG.info("cancel cloud warm up job {}, err {}", jobId, errMsg); Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); - ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().getRunnableClusterSet().remove(this.cloudClusterName); + ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().notifyJobStop(this); return true; - } private void runPendingJob() throws DdlException { Preconditions.checkState(jobState == JobState.PENDING, jobState); - // Todo: nothing to prepare yet + // make sure only one job runs concurrently for one destination cluster + if (!((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().tryRegisterRunningJob(this)) { + return; + } + // Todo: nothing to prepare yet + this.setJobDone = false; + this.lastBatchId = -1; + this.startTimeMs = System.currentTimeMillis(); + MetricRepo.updateClusterWarmUpJobLatestStartTime(String.valueOf(jobId), srcClusterName, + dstClusterName, startTimeMs); + this.fetchBeToTabletIdBatches(); + long totalTablets = beToTabletIdBatches.values().stream() + .flatMap(List::stream) + .mapToLong(List::size) + .sum(); + MetricRepo.increaseClusterWarmUpJobRequestedTablets(dstClusterName, totalTablets); + MetricRepo.increaseClusterWarmUpJobExecCount(dstClusterName); this.jobState = JobState.RUNNING; Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); LOG.info("transfer cloud warm up job {} state to {}", jobId, this.jobState); @@ -380,20 +620,67 @@ private List buildJobMetas(long beId, long batchId) { jobMeta.setDownloadType(TDownloadType.S3); jobMeta.setTabletIds(tabletIds); jobMetas.add(jobMeta); + MetricRepo.increaseClusterWarmUpJobFinishedTablets(dstClusterName, tabletIds.size()); } return jobMetas; } + private TWarmUpEventType getTWarmUpEventType() { + switch (syncEvent) { + case LOAD: + return TWarmUpEventType.LOAD; + case QUERY: + return TWarmUpEventType.QUERY; + default: + return null; + } + } + + private void runEventDrivenJob() throws Exception { + try { + initClients(); + for (Map.Entry entry : beToClient.entrySet()) { + TWarmUpTabletsRequest request = new TWarmUpTabletsRequest(); + request.setType(TWarmUpTabletsRequestType.SET_JOB); + request.setJobId(jobId); + TWarmUpEventType event = getTWarmUpEventType(); + if (event == null) { + throw new IllegalArgumentException("Unknown SyncEvent " + syncEvent); + } + request.setEvent(event); + LOG.debug("send warm up request to BE {}. job_id={}, event={}, request_type=SET_JOB(EVENT)", + entry.getKey(), jobId, syncEvent); + TWarmUpTabletsResponse response = entry.getValue().warmUpTablets(request); + if (response.getStatus().getStatusCode() != TStatusCode.OK) { + if (!response.getStatus().getErrorMsgs().isEmpty()) { + errMsg = response.getStatus().getErrorMsgs().get(0); + } + LOG.warn("send warm up request failed. job_id={}, event={}, err={}", + jobId, syncEvent, errMsg); + } + } + } catch (Exception e) { + LOG.warn("send warm up request job_id={} failed with exception {}", + jobId, e); + } finally { + releaseClients(); + } + } + private void runRunningJob() throws Exception { Preconditions.checkState(jobState == JobState.RUNNING, jobState); if (FeConstants.runningUnitTest) { Thread.sleep(1000); this.jobState = JobState.FINISHED; this.finishedTimeMs = System.currentTimeMillis(); - ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().getRunnableClusterSet().remove(this.cloudClusterName); + ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().notifyJobStop(this); Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); return; } + if (this.isEventDriven()) { + runEventDrivenJob(); + return; + } boolean changeToCancelState = false; try { initClients(); @@ -406,8 +693,9 @@ private void runRunningJob() throws Exception { request.setJobId(jobId); request.setBatchId(lastBatchId + 1); request.setJobMetas(buildJobMetas(entry.getKey(), request.batch_id)); - LOG.info("send warm up request. job_id={}, batch_id={}, job_sizes={}, request_type=SET_JOB", - jobId, request.batch_id, request.job_metas.size()); + LOG.info("send warm up request to BE {}. job_id={}, batch_id={}" + + ", job_size={}, request_type=SET_JOB", + entry.getKey(), jobId, request.batch_id, request.job_metas.size()); TWarmUpTabletsResponse response = entry.getValue().warmUpTablets(request); if (response.getStatus().getStatusCode() != TStatusCode.OK) { if (!response.getStatus().getErrorMsgs().isEmpty()) { @@ -422,7 +710,8 @@ private void runRunningJob() throws Exception { for (Map.Entry entry : beToClient.entrySet()) { TWarmUpTabletsRequest request = new TWarmUpTabletsRequest(); request.setType(TWarmUpTabletsRequestType.GET_CURRENT_JOB_STATE_AND_LEASE); - LOG.info("send warm up request. request_type=GET_CURRENT_JOB_STATE_AND_LEASE"); + LOG.info("send warm up request to BE {}. job_id={}, request_type=GET_CURRENT_JOB_STATE_AND_LEASE", + entry.getKey(), jobId); TWarmUpTabletsResponse response = entry.getValue().warmUpTablets(request); if (response.getStatus().getStatusCode() != TStatusCode.OK) { if (!response.getStatus().getErrorMsgs().isEmpty()) { @@ -434,6 +723,12 @@ private void runRunningJob() throws Exception { allLastBatchDone = false; break; } + // /api/debug_point/add/CloudWarmUpJob.FakeLastBatchNotDone + if (DebugPointUtil.isEnable("CloudWarmUpJob.FakeLastBatchNotDone")) { + allLastBatchDone = false; + LOG.info("DebugPoint:CloudWarmUpJob.FakeLastBatchNotDone, jobID={}", jobId); + break; + } } if (!changeToCancelState && allLastBatchDone) { if (retry) { @@ -454,9 +749,9 @@ private void runRunningJob() throws Exception { if (!request.job_metas.isEmpty()) { // check all batches is done or not allBatchesDone = false; - LOG.info("send warm up request. job_id={}, batch_id={}" - + "job_sizes={}, request_type=SET_BATCH", - jobId, request.batch_id, request.job_metas.size()); + LOG.info("send warm up request to BE {}. job_id={}, batch_id={}" + + ", job_size={}, request_type=SET_BATCH", + entry.getKey(), jobId, request.batch_id, request.job_metas.size()); TWarmUpTabletsResponse response = entry.getValue().warmUpTablets(request); if (response.getStatus().getStatusCode() != TStatusCode.OK) { if (!response.getStatus().getErrorMsgs().isEmpty()) { @@ -467,38 +762,23 @@ private void runRunningJob() throws Exception { } } if (allBatchesDone) { - // release job - this.jobState = JobState.FINISHED; - for (Map.Entry entry : beToClient.entrySet()) { - TWarmUpTabletsRequest request = new TWarmUpTabletsRequest(); - request.setType(TWarmUpTabletsRequestType.CLEAR_JOB); - request.setJobId(jobId); - LOG.info("send warm up request. request_type=CLEAR_JOB"); - entry.getValue().warmUpTablets(request); - } + clearJobOnBEs(); this.finishedTimeMs = System.currentTimeMillis(); - releaseClients(); - ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr() - .getRunnableClusterSet().remove(this.cloudClusterName); + if (this.isPeriodic()) { + // wait for next schedule + this.jobState = JobState.PENDING; + } else { + // release job + this.jobState = JobState.FINISHED; + } + ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().notifyJobStop(this); Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); } } } if (changeToCancelState) { // release job - this.jobState = JobState.CANCELLED; - for (Map.Entry entry : beToClient.entrySet()) { - TWarmUpTabletsRequest request = new TWarmUpTabletsRequest(); - request.setType(TWarmUpTabletsRequestType.CLEAR_JOB); - request.setJobId(jobId); - LOG.info("send warm up request. request_type=CLEAR_JOB"); - entry.getValue().warmUpTablets(request); - } - this.finishedTimeMs = System.currentTimeMillis(); - releaseClients(); - ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr() - .getRunnableClusterSet().remove(this.cloudClusterName); - Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); + cancel("job fail", false); } } catch (Exception e) { retryTime++; @@ -507,12 +787,7 @@ private void runRunningJob() throws Exception { LOG.warn("warm up job {} exception: {}", jobId, e.getMessage()); } else { // retry three times and release job - this.jobState = JobState.CANCELLED; - this.finishedTimeMs = System.currentTimeMillis(); - this.errMsg = "retry the warm up job until max retry time " + String.valueOf(maxRetryTime); - ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr() - .getRunnableClusterSet().remove(this.cloudClusterName); - Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); + cancel("retry the warm up job until max retry time " + String.valueOf(maxRetryTime), false); } releaseClients(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java b/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java index 9e57f087dbf7f8..20d4f145ec106a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java @@ -38,6 +38,13 @@ public class CloudMetrics { protected static AutoMappedMetric> CLUSTER_BACKEND_ALIVE; protected static AutoMappedMetric> CLUSTER_BACKEND_ALIVE_TOTAL; + protected static AutoMappedMetric CLUSTER_WARM_UP_JOB_EXEC_COUNT; + protected static AutoMappedMetric CLUSTER_WARM_UP_JOB_REQUESTED_TABLETS; + protected static AutoMappedMetric CLUSTER_WARM_UP_JOB_FINISHED_TABLETS; + + protected static AutoMappedMetric CLUSTER_WARM_UP_JOB_LATEST_START_TIME; + protected static AutoMappedMetric CLUSTER_WARM_UP_JOB_LAST_FINISH_TIME; + protected static void init() { if (Config.isNotCloudMode()) { return; @@ -74,5 +81,22 @@ protected static void init() { + clusterId, "cluster_name=" + clusterName); return MetricRepo.METRIC_REGISTER.histogram(metricName); }); + + CLUSTER_WARM_UP_JOB_EXEC_COUNT = new AutoMappedMetric<>(name -> new LongCounterMetric( + "file_cache_warm_up_job_exec_count", MetricUnit.NOUNIT, "warm up job execution count")); + CLUSTER_WARM_UP_JOB_LATEST_START_TIME = new AutoMappedMetric<>(name -> new LongCounterMetric( + "file_cache_warm_up_job_latest_start_time", MetricUnit.MILLISECONDS, + "the latest start time (ms, epoch time) of the warm up job")); + CLUSTER_WARM_UP_JOB_LAST_FINISH_TIME = new AutoMappedMetric<>(name -> new LongCounterMetric( + "file_cache_warm_up_job_last_finish_time", MetricUnit.MILLISECONDS, + "the last finish time (ms, epoch time) of the warm up job")); + + CLUSTER_WARM_UP_JOB_REQUESTED_TABLETS = new AutoMappedMetric<>( + name -> new LongCounterMetric("file_cache_warm_up_job_requested_tablets", + MetricUnit.NOUNIT, "warm up job requested tablets")); + + CLUSTER_WARM_UP_JOB_FINISHED_TABLETS = new AutoMappedMetric<>( + name -> new LongCounterMetric("file_cache_warm_up_job_finished_tablets", + MetricUnit.NOUNIT, "warm up job finished tablets")); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java b/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java index 64e973f78342a5..21e5e366ebccef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -105,7 +106,10 @@ public void removeMetricsByNameAndLabels(String name, List labels) private static String computeLabelId(List labels) { TreeMap labelMap = new TreeMap<>(); for (MetricLabel label : labels) { - labelMap.put(label.getKey(), label.getValue().replace("\\", "\\\\").replace("\"", "\\\"")); + labelMap.put(label.getKey(), + Optional.ofNullable(label.getValue()) + .map(v -> v.replace("\\", "\\\\").replace("\"", "\\\"")) + .orElse("")); } return labelMap.entrySet() .stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java index 4e3257c3822d8b..3d91406a98b2a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java @@ -998,6 +998,20 @@ public static void registerCloudMetrics(String clusterId, String clusterName) { queryErrCounter.setLabels(labels); MetricRepo.DORIS_METRIC_REGISTER.addMetrics(queryErrCounter); + LongCounterMetric warmUpJobExecCounter = CloudMetrics.CLUSTER_WARM_UP_JOB_EXEC_COUNT.getOrAdd(clusterId); + warmUpJobExecCounter.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(warmUpJobExecCounter); + + LongCounterMetric warmUpJobRequestedTablets = + CloudMetrics.CLUSTER_WARM_UP_JOB_REQUESTED_TABLETS.getOrAdd(clusterId); + warmUpJobRequestedTablets.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(warmUpJobRequestedTablets); + + LongCounterMetric warmUpJobFinishedTablets = + CloudMetrics.CLUSTER_WARM_UP_JOB_FINISHED_TABLETS.getOrAdd(clusterId); + warmUpJobFinishedTablets.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(warmUpJobFinishedTablets); + GaugeMetricImpl requestPerSecondGauge = CloudMetrics.CLUSTER_REQUEST_PER_SECOND_GAUGE .getOrAdd(clusterId); requestPerSecondGauge.setLabels(labels); @@ -1069,6 +1083,90 @@ public static void increaseClusterQueryErr(String clusterName) { MetricRepo.DORIS_METRIC_REGISTER.addMetrics(counter); } + public static void increaseClusterWarmUpJobExecCount(String clusterName) { + if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(clusterName)) { + return; + } + String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getCloudClusterNameToId().get(clusterName); + if (Strings.isNullOrEmpty(clusterId)) { + return; + } + LongCounterMetric counter = CloudMetrics.CLUSTER_WARM_UP_JOB_EXEC_COUNT.getOrAdd(clusterId); + List labels = new ArrayList<>(); + counter.increase(1L); + labels.add(new MetricLabel("cluster_id", clusterId)); + labels.add(new MetricLabel("cluster_name", clusterName)); + counter.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(counter); + } + + public static void increaseClusterWarmUpJobRequestedTablets(String clusterName, long tablets) { + if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(clusterName)) { + return; + } + String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getCloudClusterNameToId().get(clusterName); + if (Strings.isNullOrEmpty(clusterId)) { + return; + } + LongCounterMetric counter = CloudMetrics.CLUSTER_WARM_UP_JOB_REQUESTED_TABLETS.getOrAdd(clusterId); + List labels = new ArrayList<>(); + counter.increase(tablets); + labels.add(new MetricLabel("cluster_id", clusterId)); + labels.add(new MetricLabel("cluster_name", clusterName)); + counter.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(counter); + } + + public static void increaseClusterWarmUpJobFinishedTablets(String clusterName, long bytes) { + if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(clusterName)) { + return; + } + String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getCloudClusterNameToId().get(clusterName); + if (Strings.isNullOrEmpty(clusterId)) { + return; + } + LongCounterMetric counter = CloudMetrics.CLUSTER_WARM_UP_JOB_FINISHED_TABLETS.getOrAdd(clusterId); + List labels = new ArrayList<>(); + counter.increase(bytes); + labels.add(new MetricLabel("cluster_id", clusterId)); + labels.add(new MetricLabel("cluster_name", clusterName)); + counter.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(counter); + } + + public static void updateClusterWarmUpJobLatestStartTime( + String jobId, String srcClusterName, String dstClusterName, long timeMs) { + if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(jobId)) { + return; + } + LongCounterMetric time = CloudMetrics.CLUSTER_WARM_UP_JOB_LATEST_START_TIME.getOrAdd(jobId); + List labels = new ArrayList<>(); + time.update(timeMs); + labels.add(new MetricLabel("job_id", jobId)); + labels.add(new MetricLabel("src_cluster_name", srcClusterName)); + labels.add(new MetricLabel("dst_cluster_name", dstClusterName)); + time.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(time); + } + + public static void updateClusterWarmUpJobLastFinishTime( + String jobId, String srcClusterName, String dstClusterName, long timeMs) { + if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(jobId)) { + return; + } + LongCounterMetric time = CloudMetrics.CLUSTER_WARM_UP_JOB_LAST_FINISH_TIME.getOrAdd(jobId); + List labels = new ArrayList<>(); + time.update(timeMs); + labels.add(new MetricLabel("job_id", jobId)); + labels.add(new MetricLabel("src_cluster_name", srcClusterName)); + labels.add(new MetricLabel("dst_cluster_name", dstClusterName)); + time.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(time); + } + public static void updateClusterRequestPerSecond(String clusterId, double value, List labels) { if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(clusterId)) { return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 1217ece25a2b68..b4339023279fcd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -47,9 +47,13 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletMeta; import org.apache.doris.catalog.View; +import org.apache.doris.cloud.CloudWarmUpJob; +import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.catalog.CloudPartition; +import org.apache.doris.cloud.catalog.CloudReplica; import org.apache.doris.cloud.catalog.CloudTablet; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; +import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.AuthenticationException; @@ -2772,6 +2776,21 @@ public TGetTabletReplicaInfosResult getTabletReplicaInfos(TGetTabletReplicaInfos TGetTabletReplicaInfosResult result = new TGetTabletReplicaInfosResult(); List tabletIds = request.getTabletIds(); Map> tabletReplicaInfos = Maps.newHashMap(); + String clusterId = ""; + if (Config.isCloudMode() && request.isSetWarmUpJobId()) { + CloudWarmUpJob job = ((CloudEnv) Env.getCurrentEnv()) + .getCacheHotspotMgr() + .getCloudWarmUpJob(request.getWarmUpJobId()); + if (job == null) { + LOG.info("warmup job {} is not running, notify caller BE {} to cancel job", + job.getJobId(), clientAddr); + // notify client to cancel this job + result.setStatus(new TStatus(TStatusCode.CANCELLED)); + return result; + } + clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getCloudClusterIdByName(job.getDstClusterName()); + } for (Long tabletId : tabletIds) { if (DebugPointUtil.isEnable("getTabletReplicaInfos.returnEmpty")) { LOG.info("enable getTabletReplicaInfos.returnEmpty"); @@ -2781,11 +2800,17 @@ public TGetTabletReplicaInfosResult getTabletReplicaInfos(TGetTabletReplicaInfos List replicas = Env.getCurrentEnv().getCurrentInvertedIndex() .getReplicasByTabletId(tabletId); for (Replica replica : replicas) { - if (!replica.isNormal()) { + if (!replica.isNormal() && !request.isSetWarmUpJobId()) { LOG.warn("replica {} not normal", replica.getId()); continue; } - Backend backend = Env.getCurrentSystemInfo().getBackend(replica.getBackendIdWithoutException()); + Backend backend; + if (Config.isCloudMode() && request.isSetWarmUpJobId()) { + CloudReplica cloudReplica = (CloudReplica) replica; + backend = cloudReplica.getPrimaryBackend(clusterId); + } else { + backend = Env.getCurrentSystemInfo().getBackend(replica.getBackendIdWithoutException()); + } if (backend != null) { TReplicaInfo replicaInfo = new TReplicaInfo(); replicaInfo.setHost(backend.getHost()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyCloudWarmUpJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyCloudWarmUpJobTest.java index 0803fee02b6766..bca8c1eb0273e3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyCloudWarmUpJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyCloudWarmUpJobTest.java @@ -56,6 +56,7 @@ public void testSerialization() throws IOException { long createTimeMs = 11111; String errMsg = "testMsg"; long finishedTimesMs = 22222; + String srcClusterName = "cloudSrc"; String clusterName = "cloudTest"; long lastBatchId = 33333; Map>> beToTabletIdBatches = new HashMap<>(); @@ -68,7 +69,7 @@ public void testSerialization() throws IOException { beToThriftAddress.put(998L, "address"); CloudWarmUpJob.JobType jobType = JobType.TABLE; - CloudWarmUpJob warmUpJob = new CloudWarmUpJob(jobId, clusterName, beToTabletIdBatches, jobType); + CloudWarmUpJob warmUpJob = new CloudWarmUpJob(jobId, srcClusterName, clusterName, beToTabletIdBatches, jobType); warmUpJob.setJobState(jobState); warmUpJob.setCreateTimeMs(createTimeMs); warmUpJob.setErrMsg(errMsg); @@ -93,7 +94,7 @@ public void testSerialization() throws IOException { Assert.assertEquals(createTimeMs, warmUpJob2.getCreateTimeMs()); Assert.assertEquals(errMsg, warmUpJob2.getErrMsg()); Assert.assertEquals(finishedTimesMs, warmUpJob2.getFinishedTimeMs()); - Assert.assertEquals(clusterName, warmUpJob2.getCloudClusterName()); + Assert.assertEquals(clusterName, warmUpJob2.getDstClusterName()); Assert.assertEquals(lastBatchId, warmUpJob2.getLastBatchId()); Map>> beToTabletIdBatches2 = warmUpJob2.getBeToTabletIdBatches(); Assert.assertEquals(1, beToTabletIdBatches2.size()); diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 150c07fcf9a9ed..5d7b653d9607f8 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -837,6 +837,28 @@ message PGetFileCacheMetaResponse { repeated FileCacheBlockMeta file_cache_block_metas = 1; } +message PWarmUpRowsetRequest { + repeated RowsetMetaPB rowset_metas = 1; + optional int64 unix_ts_us = 2; +} + +message PWarmUpRowsetResponse { +} + +message RecycleCacheMeta { + optional int64 tablet_id = 1; + optional string rowset_id = 2; + optional int64 num_segments = 3; + repeated string index_file_names = 4; +} + +message PRecycleCacheRequest { + repeated RecycleCacheMeta cache_metas = 1; +} + +message PRecycleCacheResponse { +} + message PReportStreamLoadStatusRequest { optional PUniqueId load_id = 1; optional PStatus status = 2; @@ -1029,6 +1051,8 @@ service PBackendService { rpc fetch_table_schema(PFetchTableSchemaRequest) returns (PFetchTableSchemaResult); rpc multiget_data(PMultiGetRequest) returns (PMultiGetResponse); rpc get_file_cache_meta_by_tablet_id(PGetFileCacheMetaRequest) returns (PGetFileCacheMetaResponse); + rpc warm_up_rowset(PWarmUpRowsetRequest) returns (PWarmUpRowsetResponse); + rpc recycle_cache(PRecycleCacheRequest) returns (PRecycleCacheResponse); rpc tablet_fetch_data(PTabletKeyLookupRequest) returns (PTabletKeyLookupResponse); rpc get_column_ids_by_tablet_ids(PFetchColIdsRequest) returns (PFetchColIdsResponse); rpc get_tablet_rowset_versions(PGetTabletVersionsRequest) returns (PGetTabletVersionsResponse); diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift index 7895d8b83fa0ab..7cd09dc5cf7534 100644 --- a/gensrc/thrift/BackendService.thrift +++ b/gensrc/thrift/BackendService.thrift @@ -186,11 +186,16 @@ enum TDownloadType { S3 = 1, } +enum TWarmUpEventType { + LOAD = 0, + QUERY = 1, +} + enum TWarmUpTabletsRequestType { SET_JOB = 0, SET_BATCH = 1, GET_CURRENT_JOB_STATE_AND_LEASE = 2, - CLEAR_JOB = 3, + CLEAR_JOB = 3 } struct TJobMeta { @@ -205,6 +210,7 @@ struct TWarmUpTabletsRequest { 2: required i64 batch_id 3: optional list job_metas 4: required TWarmUpTabletsRequestType type + 5: optional TWarmUpEventType event } struct TWarmUpTabletsResponse { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 2d7779bb08a59c..b9f2847e1687ec 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1375,6 +1375,7 @@ struct TGetBinlogResult { struct TGetTabletReplicaInfosRequest { 1: required list tablet_ids + 2: optional i64 warm_up_job_id } struct TGetTabletReplicaInfosResult { diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy index 3d22b75e98dfcf..b6ae1be881e2fa 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_cluster") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy index f9a5004a84e370..9a2aff3393333f 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy @@ -20,7 +20,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_warm_up_cluster_batch") { def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy index e9be62cf9821ee..4f97116e120e7e 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_cluster_bigsize") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy index bf3121b269f6e3..d86238c5be9ad4 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_cluster_empty") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event.groovy new file mode 100644 index 00000000000000..60ae1819395ffe --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event.groovy @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(10000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + clearFileCacheOnAllBackends() + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_add_new_be.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_add_new_be.groovy new file mode 100644 index 00000000000000..3bb78723b23663 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_add_new_be.groovy @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_add_new_be', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + // Add new backends to cluster 2 + cluster.addBackend(2, clusterName2) + + clearFileCacheOnAllBackends() + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_active.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_active.groovy new file mode 100644 index 00000000000000..65ef46be4c84d1 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_active.groovy @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_cancel_active', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(10000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def getClusterTTLCacheSizeSum = { cluster -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + long sum = 0 + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def size = getTTLCacheSize(ip, port) + sum += size + logger.info("be be ${ip}:${port} ttl cache size ${size}") + } + + return sum + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def srcSum = getClusterTTLCacheSizeSum(cluster1) + def dstSum = getClusterTTLCacheSizeSum(cluster2) + + logger.info("ttl_cache_size: src=${srcSum} dst=${dstSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, dstSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + clearFileCacheOnAllBackends() + + // First make some entries in tablet location info ttl cache + sleep(15000) + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + def cacheSize0 = getClusterTTLCacheSizeSum(clusterName2) + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + sleep(5000) + + // At this point, cache should be expired, so we expect no more syncs + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + def cacheSize1 = getClusterTTLCacheSizeSum(clusterName1); + def cacheSize2 = getClusterTTLCacheSizeSum(clusterName2); + assertTrue(cacheSize1 > cacheSize0, "cache size in src cluster should increase") + assertEquals(cacheSize0, cacheSize2, "no more syncs after job cancel is expected") + + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy new file mode 100644 index 00000000000000..5c1cdb8ce0594f --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy @@ -0,0 +1,248 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_cancel', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + 'warmup_tablet_replica_info_cache_ttl_sec=10', + ] + options.cloudMode = true + + def setDebugPoint = {ip, port, op, name -> + def urlStr = "http://${ip}:${port}/api/debug_point/${op}/${name}" + def url = new URL(urlStr) + def conn = url.openConnection() + conn.requestMethod = 'POST' + conn.doOutput = true + + // Send empty body (required to trigger POST) + conn.outputStream.withWriter { it << "" } + + // Read response + def responseText = conn.inputStream.text + def json = new JsonSlurper().parseText(responseText) + + return json?.msg == "OK" && json?.code == 0 + } + + def setDebugPointsForCluster = { cluster, debug_point, enable -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[4] + if (enable) { + assertTrue(setDebugPoint(ip, port, 'add', debug_point)) + } else { + assertTrue(setDebugPoint(ip, port, 'remove', debug_point)) + } + } + } + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(10000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def getClusterTTLCacheSizeSum = { cluster -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + long sum = 0 + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def size = getTTLCacheSize(ip, port) + sum += size + logger.info("be be ${ip}:${port} ttl cache size ${size}") + } + + return sum + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def srcSum = getClusterTTLCacheSizeSum(cluster1) + def dstSum = getClusterTTLCacheSizeSum(cluster2) + + logger.info("ttl_cache_size: src=${srcSum} dst=${dstSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, dstSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + clearFileCacheOnAllBackends() + + // First make some entries in tablet location info ttl cache + sleep(15000) + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + def cacheSize0 = getClusterTTLCacheSizeSum(clusterName2) + + // Make BE ignore the cancel request + setDebugPointsForCluster(clusterName1, 'CloudWarmUpManager.set_event.ignore_all', true) + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Initially, the cache has not expired, so we expect some more syncs + for (int i = 0; i < 10; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + def cacheSize1 = getClusterTTLCacheSizeSum(clusterName2); + assertTrue(cacheSize1 > cacheSize0, "some more syncs before cache expire is expected") + + // At this point, cache should be expired, so we expect no more syncs + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + def cacheSize2 = getClusterTTLCacheSizeSum(clusterName2); + assertEquals(cacheSize1, cacheSize2, "no more syncs after cache expire is expected") + + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + + setDebugPointsForCluster(clusterName1, 'CloudWarmUpManager.set_event.ignore_all', false) + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_compaction.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_compaction.groovy new file mode 100644 index 00000000000000..697bffeabfa931 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_compaction.groovy @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_compaction', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + def wait_for_latest_op_on_table_finish = { tableName, OpTimeout -> + def delta_time = 1000 + def useTime = 0 + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + def alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(tableName + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def run_compaction = { + def timeout = 60000 + sql """insert into test values (1, '{"a" : 1.0}')""" + sql """insert into test values (2, '{"a" : 111.1111}')""" + sql """insert into test values (3, '{"a" : "11111"}')""" + sql """insert into test values (4, '{"a" : 1111111111}')""" + sql """insert into test values (5, '{"a" : 1111.11111}')""" + sql """insert into test values (6, '{"a" : "11111"}')""" + sql """insert into test values (7, '{"a" : 11111.11111}')""" + sql "alter table test modify column col1 variant;" + wait_for_latest_op_on_table_finish("test", timeout) + sql """insert into test values (1, '{"a" : 1.0}')""" + sql """insert into test values (2, '{"a" : 111.1111}')""" + sql """insert into test values (3, '{"a" : "11111"}')""" + sql """insert into test values (4, '{"a" : 1111111111}')""" + sql """insert into test values (5, '{"a" : 1111.11111}')""" + sql """insert into test values (6, '{"a" : "11111"}')""" + sql """insert into test values (7, '{"a" : 11111.11111}')""" + trigger_and_wait_compaction("test", "cumulative") + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + sql """ + create table test ( + col0 int not null, + col1 variant NOT NULL + ) UNIQUE KEY(`col0`) + DISTRIBUTED BY HASH(col0) BUCKETS 1 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "false"); + """ + + clearFileCacheOnAllBackends() + sleep(15000) + + run_compaction(); + sleep(15000) + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy new file mode 100644 index 00000000000000..fc3ac52ae91d29 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy @@ -0,0 +1,251 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_rename', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + 'warmup_tablet_replica_info_cache_ttl_sec=1' + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(10000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def getClusterTTLCacheSizeSum = { cluster -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + long sum = 0 + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def size = getTTLCacheSize(ip, port) + sum += size + logger.info("be be ${ip}:${port} ttl cache size ${size}") + } + + return sum + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + def clusterName3 = "warmup_other" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + clearFileCacheOnAllBackends() + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + srcSumOld = getClusterTTLCacheSizeSum(clusterName1) + dstSumOld = getClusterTTLCacheSizeSum(clusterName2) + + // rename + sql """ALTER SYSTEM RENAME COMPUTE GROUP ${clusterName2} ${clusterName3}""" + sleep(5000) + + assertEquals(0, getClusterTTLCacheSizeSum(clusterName2)) + assertEquals(dstSumOld, getClusterTTLCacheSizeSum(clusterName3)) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + + assertEquals(0, getClusterTTLCacheSizeSum(clusterName2)) + assertEquals(dstSumOld, getClusterTTLCacheSizeSum(clusterName3)) + + // rename back + sql """ALTER SYSTEM RENAME COMPUTE GROUP ${clusterName3} ${clusterName2}""" + clearFileCacheOnAllBackends() + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_all_be.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_all_be.groovy new file mode 100644 index 00000000000000..1e2178af1fb69f --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_all_be.groovy @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_restart_all_be', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + def restartAllBE = { + cluster.restartBackends() + sleep(5000) + + // wait for be restart + boolean ok = false + int cnt = 0 + for (; !ok && cnt < 30; cnt++) { + ok = true; + sql_return_maparray("show backends").each { be -> + if (!be.Alive.toBoolean()) { + ok = false + } + } + //def be = sql_return_maparray("show backends").get(0) + logger.info("wait for BE restart...") + Thread.sleep(1000) + } + if (!ok) { + logger.info("BE failed to restart") + assertTrue(false) + } + sleep(3000) + } + + restartAllBE() + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_master_fe.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_master_fe.groovy new file mode 100644 index 00000000000000..f96c0f0b4135bf --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_master_fe.groovy @@ -0,0 +1,218 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_restart_master_fe', 'docker') { + def options = new ClusterOptions() + options.feNum = 3 + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + def restartMasterFe = { + def oldMasterFe = cluster.getMasterFe() + cluster.restartFrontends(oldMasterFe.index) + boolean hasRestart = false + for (int i = 0; i < 30; i++) { + if (cluster.getFeByIndex(oldMasterFe.index).alive) { + hasRestart = true + break + } + sleep 1000 + } + assertTrue(hasRestart) + context.reconnectFe() + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + restartMasterFe() + sql """use @${clusterName1}""" + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_schema_change.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_schema_change.groovy new file mode 100644 index 00000000000000..bc426a16d52f9a --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_schema_change.groovy @@ -0,0 +1,266 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_schema_change', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + def wait_for_latest_op_on_table_finish = { tableName, OpTimeout -> + def delta_time = 1000 + def useTime = 0 + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + def alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(tableName + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def run_schema_change = { + def table_name = "test" + def timeout = 60000 + def delta_time = 1000 + def useTime = 0 + + // sql "set experimental_enable_nereids_planner = true" + // add, drop columns + sql """INSERT INTO ${table_name} SELECT *, '{"k1":1, "k2": "hello world", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]]}' FROM numbers("number" = "4096")""" + sql "alter table ${table_name} add column v2 variant default null" + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name}""" + sql "alter table ${table_name} drop column v2" + sql """INSERT INTO ${table_name} SELECT k, v from ${table_name}""" + sql """select v['k1'] from ${table_name} order by k limit 10""" + sql "alter table ${table_name} add column vs string default null" + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name}""" + sql """select v['k1'] from ${table_name} order by k desc limit 10""" + sql """select v['k1'], cast(v['k2'] as string) from ${table_name} order by k desc limit 10""" + + // sql "set experimental_enable_nereids_planner = true" + // add, drop index + sql "alter table ${table_name} add index btm_idxk (k) using bitmap ;" + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name}""" + wait_for_latest_op_on_table_finish(table_name, timeout) + + // drop column is linked schema change + sql "drop index btm_idxk on ${table_name};" + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name} limit 1024""" + wait_for_latest_op_on_table_finish(table_name, timeout) + sql """select v['k1'] from ${table_name} order by k desc limit 10""" + sql """select v['k1'], cast(v['k2'] as string) from ${table_name} order by k desc limit 10""" + + // add, drop materialized view + createMV("""create materialized view var_order as select vs, k, v from ${table_name} order by vs""") + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name} limit 4096""" + createMV("""create materialized view var_cnt as select k, count(k) from ${table_name} group by k""") + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name} limit 8101""" + sql """DROP MATERIALIZED VIEW var_cnt ON ${table_name}""" + sql """INSERT INTO ${table_name} SELECT k, v,v from ${table_name} limit 1111""" + // select from mv + sql """select v['k1'], cast(v['k2'] as string) from ${table_name} order by k desc limit 10""" + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + def table_name = "test" + sql "DROP TABLE IF EXISTS ${table_name}" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 4 + properties("file_cache_ttl_seconds" = "3600"); + """ + + clearFileCacheOnAllBackends() + sleep(20000) + + run_schema_change(); + sleep(15000) + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy new file mode 100644 index 00000000000000..87938bd60af998 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_periodic', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + def size = getTTLCacheSize(ip, port) + srcSum += size + logger.info("src be ${ip}:${port} ttl cache size ${size}") + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + def size = getTTLCacheSize(ip, port) + tgtSum += size + logger.info("dst be ${ip}:${port} ttl cache size ${size}") + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + + clearFileCacheOnAllBackends() + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + sleep(5000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy new file mode 100644 index 00000000000000..5c58da8a564410 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_periodic_add_new_be', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + // Add new backends to cluster 2 + cluster.addBackend(2, clusterName2); + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + clearFileCacheOnAllBackends() + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + sleep(15000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_and_event.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_and_event.groovy new file mode 100644 index 00000000000000..156c3e1697ae8b --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_and_event.groovy @@ -0,0 +1,237 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_periodic_and_event', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job 1 + def jobId1_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + def jobId1 = jobId1_[0][0] + logger.info("Periodic warm-up job ID: ${jobId1}") + + // Start warm up job 2 + def jobId2_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId2 = jobId2_[0][0] + logger.info("Event driven warm-up job ID: ${jobId2}") + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + clearFileCacheOnAllBackends() + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + sleep(15000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId1}""" + assertEquals(jobInfo[0][0], jobId1) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + def jobInfo2 = sql """SHOW WARM UP JOB WHERE ID = ${jobId2}""" + assertEquals(jobInfo2[0][0], jobId2) + assertEquals(jobInfo2[0][1], clusterName1) + assertEquals(jobInfo2[0][2], clusterName2) + assertEquals(jobInfo2[0][4], "CLUSTER") + assertTrue(jobInfo2[0][3] in ["RUNNING", "PENDING"]) + assertEquals(jobInfo2[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId1}""" + def cancelInfo1 = sql """SHOW WARM UP JOB WHERE ID = ${jobId1}""" + assertEquals(cancelInfo1[0][3], "CANCELLED") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId2}""" + def cancelInfo2 = sql """SHOW WARM UP JOB WHERE ID = ${jobId2}""" + assertEquals(cancelInfo2[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy new file mode 100644 index 00000000000000..271fec8bbfc9b0 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_periodic_rename', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def getClusterTTLCacheSizeSum = { cluster -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + long sum = 0 + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def size = getTTLCacheSize(ip, port) + sum += size + logger.info("be be ${ip}:${port} ttl cache size ${size}") + } + + return sum + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + def size = getTTLCacheSize(ip, port) + srcSum += size + logger.info("src be ${ip}:${port} ttl cache size ${size}") + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + def size = getTTLCacheSize(ip, port) + tgtSum += size + logger.info("dst be ${ip}:${port} ttl cache size ${size}") + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + def clusterName3 = "warmup_other" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + + clearFileCacheOnAllBackends() + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + sql """ALTER SYSTEM RENAME COMPUTE GROUP ${clusterName2} ${clusterName3}""" + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + sleep(5000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + assertTrue(getClusterTTLCacheSizeSum(clusterName1) > 0) + assertEquals(0, getClusterTTLCacheSizeSum(clusterName2)) + assertEquals(0, getClusterTTLCacheSizeSum(clusterName3)) + + sql """ALTER SYSTEM RENAME COMPUTE GROUP ${clusterName3} ${clusterName2}""" + sleep(5000) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_restart_master_fe.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_restart_master_fe.groovy new file mode 100644 index 00000000000000..d7f49d3fc82a4f --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_restart_master_fe.groovy @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_periodic_restart_master_fe', 'docker') { + def options = new ClusterOptions() + options.feNum = 3 + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + def restartMasterFe = { + def oldMasterFe = cluster.getMasterFe() + cluster.restartFrontends(oldMasterFe.index) + boolean hasRestart = false + for (int i = 0; i < 30; i++) { + if (cluster.getFeByIndex(oldMasterFe.index).alive) { + hasRestart = true + break + } + sleep 1000 + } + assertTrue(hasRestart) + context.reconnectFe() + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + restartMasterFe(); + sql """use @${clusterName1}""" + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + clearFileCacheOnAllBackends() + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + sleep(5000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_slow_job.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_slow_job.groovy new file mode 100644 index 00000000000000..5e64dddbbe0f31 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_slow_job.groovy @@ -0,0 +1,288 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +import java.time.Duration +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import java.util.Base64 + +suite('test_warm_up_cluster_periodic_slow_job', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + 'enable_debug_points=true', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def setDebugPoint = {ip, port, op, name -> + def urlStr = "http://${ip}:${port}/api/debug_point/${op}/${name}" + def url = new URL(urlStr) + def conn = url.openConnection() + conn.requestMethod = 'POST' + conn.doOutput = true + + // Add Basic Auth header + def authString = "root:" + def encodedAuth = Base64.encoder.encodeToString(authString.getBytes("UTF-8")) + conn.setRequestProperty("Authorization", "Basic ${encodedAuth}") + + // Send empty body (required to trigger POST) + conn.outputStream.withWriter { it << "" } + + // Read response + def responseText = conn.inputStream.text + def json = new JsonSlurper().parseText(responseText) + + return json?.msg == "OK" && json?.code == 0 + } + + def addDebugPoint = { ip, http_port, name -> + return setDebugPoint(ip, http_port, 'add', name) + } + + def removeDebugPoint = { ip, http_port, name -> + return setDebugPoint(ip, http_port, 'remove', name) + } + + def addFEDebugPoint = { name -> + def fe = cluster.getMasterFe() + return addDebugPoint(fe.host, fe.httpPort, name) + } + + def removeFEDebugPoint = { name -> + def fe = cluster.getMasterFe() + return removeDebugPoint(fe.host, fe.httpPort, name) + } + + def getWarmUpJobInfo = { jobId -> + def jobInfo = sql """SHOW WARM UP JOB WHERE id = ${jobId}""" + + if (jobInfo == null || jobInfo.isEmpty()) { + fail("No warm up job found with ID: ${jobId}") + } + + def job = jobInfo[0] + def formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS") + def sTime = LocalDateTime.parse(job[7], formatter) + def cTime = LocalDateTime.parse(job[6], formatter) + + return [ + status : job[3], + createTime: cTime, + startTime : sTime, + ] + } + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + def size = getTTLCacheSize(ip, port) + srcSum += size + logger.info("src be ${ip}:${port} ttl cache size ${size}") + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + def size = getTTLCacheSize(ip, port) + tgtSum += size + logger.info("dst be ${ip}:${port} ttl cache size ${size}") + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + + clearFileCacheOnAllBackends() + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + + addFEDebugPoint("CloudWarmUpJob.FakeLastBatchNotDone") + + def sync_sec = 1 + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "${sync_sec}" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + def wait_sec = sync_sec * 5 + sleep(wait_sec * 2 * 1000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + def info1 = getWarmUpJobInfo(jobId) + logger.info("info1 {}", info1) + assertEquals("RUNNING", info1.status) + + sleep(wait_sec * 2 * 1000) + + def info2 = getWarmUpJobInfo(jobId) + logger.info("info2 {}", info2) + assertEquals("RUNNING", info2.status) + assertEquals(info1.createTime, info2.createTime) + assertEquals(info1.startTime, info2.startTime) + + removeFEDebugPoint("CloudWarmUpJob.FakeLastBatchNotDone") + sleep(wait_sec * 1000) + + def info3 = getWarmUpJobInfo(jobId) + logger.info("info3 {}", info3) + assertEquals(info1.createTime, info3.createTime) + assertTrue(Duration.between(info1.startTime, info3.startTime).seconds.abs() > wait_sec * 4) + + sleep(wait_sec * 1000) + def info4 = getWarmUpJobInfo(jobId) + logger.info("info4 {}", info4) + assertEquals(info1.createTime, info4.createTime) + assertTrue(Duration.between(info3.startTime, info4.startTime).seconds.abs() > sync_sec) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_repeat_jobs.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_repeat_jobs.groovy new file mode 100644 index 00000000000000..8ad210def6adbc --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_repeat_jobs.groovy @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_repeat_jobs', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.cloudMode = true + + docker(options) { + def clusterName1 = "cluster1" + def clusterName2 = "cluster2" + def clusterName3 = "cluster3" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + cluster.addBackend(1, clusterName3) + + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + def jobId = jobId_[0][0] + + logger.info("JobID = {}", jobId) + + // For periodic jobs, it's not allowed to start a job + // with same src cluster, dst cluster, and sync_mode + try { + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "10" + ) + """ + assertTrue(false, "expected exception") + } catch (java.sql.SQLException e) { + assertTrue(e.getMessage().contains("already has a runnable job"), e.getMessage()); + } + + // It's allowed to start a job with same dst cluster + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName3} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + // It's allowed to start a ONCE job with same src dst cluster + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "once" + ) + """ + + // It's allowed to start a ONCE job with same src dst cluster + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + """ + + // It's allowed to create a job in the opposite direction + sql """ + WARM UP CLUSTER ${clusterName1} WITH CLUSTER ${clusterName2} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + // after cancelling the old job, we can create another job with same attributes + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + // It's allowed to start a event_driven job alongside with a periodic job + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + // For event driven jobs, it's not allowed to start a job + // with same src cluster, dst cluster, and sync_mode + try { + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + assertTrue(false, "expected exception") + } catch (java.sql.SQLException e) { + assertTrue(e.getMessage().contains("already has a runnable job"), e.getMessage()); + } + + // For event driven jobs, it's allowed to start a job with same dst cluster + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName3} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy index a086731efffce4..5a16e92b36bb5d 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_compute_group") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy index 0eb93f2896c39d..c6819ad58ec20b 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_partition") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } List ipList = new ArrayList<>(); diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy index b7eb8761951049..258e9e87ef6121 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_table") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def getTablesFromShowCommand = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy index 77286717117578..03d45f1cce8e1e 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_tables") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def getTablesFromShowCommand = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """