From ab68556c969d5955081a24f08d8c09a297e05534 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Tue, 16 Apr 2024 21:23:06 +0800 Subject: [PATCH 1/2] 1 --- be/src/agent/task_worker_pool.cpp | 4 ++-- be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp | 4 ++-- be/src/olap/task/engine_publish_version_task.cpp | 4 ++-- be/src/olap/task/engine_publish_version_task.h | 2 +- be/src/runtime/fold_constant_executor.cpp | 2 +- be/src/runtime/fragment_mgr.cpp | 2 ++ be/src/service/backend_service.cpp | 2 +- be/src/service/internal_service.cpp | 2 +- 8 files changed, 12 insertions(+), 10 deletions(-) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 39d96bc209816c..c491c996b32cc4 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -180,7 +180,7 @@ void alter_tablet(StorageEngine& engine, const TAgentTaskRequest& agent_task_req new_tablet_id = agent_task_req.alter_tablet_req_v2.new_tablet_id; new_schema_hash = agent_task_req.alter_tablet_req_v2.new_schema_hash; auto mem_tracker = MemTrackerLimiter::create_shared( - MemTrackerLimiter::Type::SCHEMA_CHANGE, + MemTrackerLimiter::Type::OTHER, fmt::format("EngineAlterTabletTask#baseTabletId={}:newTabletId={}", std::to_string(agent_task_req.alter_tablet_req_v2.base_tablet_id), std::to_string(agent_task_req.alter_tablet_req_v2.new_tablet_id), @@ -249,7 +249,7 @@ void alter_cloud_tablet(CloudStorageEngine& engine, const TAgentTaskRequest& age if (status.ok()) { new_tablet_id = agent_task_req.alter_tablet_req_v2.new_tablet_id; auto mem_tracker = MemTrackerLimiter::create_shared( - MemTrackerLimiter::Type::SCHEMA_CHANGE, + MemTrackerLimiter::Type::OTHER, fmt::format("EngineAlterTabletTask#baseTabletId={}:newTabletId={}", std::to_string(agent_task_req.alter_tablet_req_v2.base_tablet_id), std::to_string(agent_task_req.alter_tablet_req_v2.new_tablet_id), diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp index 9c784b1ced77e7..d31e7bd64ff179 100644 --- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp +++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp @@ -42,7 +42,7 @@ CloudEngineCalcDeleteBitmapTask::CloudEngineCalcDeleteBitmapTask( _cal_delete_bitmap_req(cal_delete_bitmap_req), _error_tablet_ids(error_tablet_ids), _succ_tablet_ids(succ_tablet_ids) { - _mem_tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::SCHEMA_CHANGE, + _mem_tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, "CloudEngineCalcDeleteBitmapTask"); } @@ -134,7 +134,7 @@ CloudTabletCalcDeleteBitmapTask::CloudTabletCalcDeleteBitmapTask( _transaction_id(transaction_id), _version(version) { _mem_tracker = MemTrackerLimiter::create_shared( - MemTrackerLimiter::Type::SCHEMA_CHANGE, + MemTrackerLimiter::Type::OTHER, fmt::format("CloudTabletCalcDeleteBitmapTask#_transaction_id={}", _transaction_id)); } diff --git a/be/src/olap/task/engine_publish_version_task.cpp b/be/src/olap/task/engine_publish_version_task.cpp index 8a7ae8764d52b2..41d28a6124b1d8 100644 --- a/be/src/olap/task/engine_publish_version_task.cpp +++ b/be/src/olap/task/engine_publish_version_task.cpp @@ -82,7 +82,7 @@ EnginePublishVersionTask::EnginePublishVersionTask( _succ_tablets(succ_tablets), _discontinuous_version_tablets(discontinuous_version_tablets), _table_id_to_num_delta_rows(table_id_to_num_delta_rows) { - _mem_tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::SCHEMA_CHANGE, + _mem_tracker = MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, "TabletPublishTxnTask"); } @@ -357,7 +357,7 @@ TabletPublishTxnTask::TabletPublishTxnTask(StorageEngine& engine, _transaction_id(transaction_id), _version(version), _tablet_info(tablet_info), - _mem_tracker(MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::SCHEMA_CHANGE, + _mem_tracker(MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, "TabletPublishTxnTask")) { _stats.submit_time_us = MonotonicMicros(); } diff --git a/be/src/olap/task/engine_publish_version_task.h b/be/src/olap/task/engine_publish_version_task.h index 38e0cb1b6acd40..e4824176368c41 100644 --- a/be/src/olap/task/engine_publish_version_task.h +++ b/be/src/olap/task/engine_publish_version_task.h @@ -121,7 +121,7 @@ class AsyncTabletPublishTask { _partition_id(partition_id), _transaction_id(transaction_id), _version(version), - _mem_tracker(MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::SCHEMA_CHANGE, + _mem_tracker(MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::OTHER, "AsyncTabletPublishTask")) { _stats.submit_time_us = MonotonicMicros(); } diff --git a/be/src/runtime/fold_constant_executor.cpp b/be/src/runtime/fold_constant_executor.cpp index 5ba384baf84b46..cf7695c86f2990 100644 --- a/be/src/runtime/fold_constant_executor.cpp +++ b/be/src/runtime/fold_constant_executor.cpp @@ -160,7 +160,7 @@ Status FoldConstantExecutor::_init(const TQueryGlobals& query_globals, fragment_params.params = params; fragment_params.protocol_version = PaloInternalServiceVersion::V1; _mem_tracker = MemTrackerLimiter::create_shared( - MemTrackerLimiter::Type::SCHEMA_CHANGE, + MemTrackerLimiter::Type::OTHER, fmt::format("FoldConstant:query_id={}", print_id(_query_id))); _runtime_state = RuntimeState::create_unique(fragment_params.params, query_options, query_globals, diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index 14f77a98ace909..13a544d9220120 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -195,6 +195,7 @@ std::string FragmentMgr::to_http_path(const std::string& file_name) { Status FragmentMgr::trigger_pipeline_context_report( const ReportStatusRequest req, std::shared_ptr&& ctx) { return _async_report_thread_pool->submit_func([this, req, ctx]() { + SCOPED_ATTACH_TASK(ctx->get_query_ctx()->query_mem_tracker); coordinator_callback(req); if (!req.done) { ctx->refresh_next_report_time(); @@ -900,6 +901,7 @@ Status FragmentMgr::exec_plan_fragment(const TPipelineFragmentParams& params, return Status::OK(); } else { auto pre_and_submit = [&](int i) { + SCOPED_ATTACH_TASK_WITH_ID(query_ctx->query_mem_tracker, params.query_id); const auto& local_params = params.local_params[i]; const TUniqueId& fragment_instance_id = local_params.fragment_instance_id; diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp index b1a110144ef8c7..3f19e504e039d6 100644 --- a/be/src/service/backend_service.cpp +++ b/be/src/service/backend_service.cpp @@ -104,7 +104,7 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg* arg) { const auto& local_tablet_uid = local_tablet->tablet_uid(); std::shared_ptr mem_tracker = MemTrackerLimiter::create_shared( - MemTrackerLimiter::Type::SCHEMA_CHANGE, fmt::format("IngestBinlog#TxnId={}", txn_id)); + MemTrackerLimiter::Type::OTHER, fmt::format("IngestBinlog#TxnId={}", txn_id)); SCOPED_ATTACH_TASK(mem_tracker); auto& request = arg->request; diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index aa9c207d68946c..899162e50f766e 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -728,7 +728,7 @@ void PInternalService::fetch_table_schema(google::protobuf::RpcController* contr const TFileScanRangeParams& params = file_scan_range.params; std::shared_ptr mem_tracker = MemTrackerLimiter::create_shared( - MemTrackerLimiter::Type::SCHEMA_CHANGE, + MemTrackerLimiter::Type::OTHER, fmt::format("{}#{}", params.format_type, params.file_type)); SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(mem_tracker); From a05813267c12c2ab7f74514a9bdabd4ef698f091 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Wed, 17 Apr 2024 00:38:18 +0800 Subject: [PATCH 2/2] 2 --- be/src/runtime/fragment_mgr.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index 13a544d9220120..03d6525154db81 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -901,7 +901,6 @@ Status FragmentMgr::exec_plan_fragment(const TPipelineFragmentParams& params, return Status::OK(); } else { auto pre_and_submit = [&](int i) { - SCOPED_ATTACH_TASK_WITH_ID(query_ctx->query_mem_tracker, params.query_id); const auto& local_params = params.local_params[i]; const TUniqueId& fragment_instance_id = local_params.fragment_instance_id; @@ -984,7 +983,10 @@ Status FragmentMgr::exec_plan_fragment(const TPipelineFragmentParams& params, for (size_t i = 0; i < target_size; i++) { RETURN_IF_ERROR(_thread_pool->submit_func([&, i]() { - prepare_status[i] = pre_and_submit(i); + { + SCOPED_ATTACH_TASK_WITH_ID(query_ctx->query_mem_tracker, params.query_id); + prepare_status[i] = pre_and_submit(i); + } std::unique_lock lock(m); prepare_done++; if (prepare_done == target_size) {