From 1089cda6fd8a399abf744052344e11ff7a139528 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Fri, 18 Sep 2020 22:18:35 +0800 Subject: [PATCH 01/27] compaction memory optimize and code refactor --- be/src/common/config.h | 2 + be/src/common/configbase.h | 2 + be/src/olap/CMakeLists.txt | 1 + be/src/olap/compaction_permit_limiter.cpp | 63 +++++++++++++ be/src/olap/compaction_permit_limiter.h | 79 ++++++++++++++++ be/src/olap/olap_server.cpp | 109 +++++++++++++++++++++- be/src/olap/storage_engine.cpp | 10 +- be/src/olap/storage_engine.h | 15 ++- 8 files changed, 271 insertions(+), 10 deletions(-) create mode 100644 be/src/olap/compaction_permit_limiter.cpp create mode 100644 be/src/olap/compaction_permit_limiter.h diff --git a/be/src/common/config.h b/be/src/common/config.h index b074c85cbd640c..aa2fe41b34e34a 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -318,6 +318,8 @@ namespace config { // This config can be set to 0, which means to forbid any compaction, for some special cases. CONF_Int32(max_compaction_concurrency, "-1"); + CONF_uInt32(total_permits_memory_for_compaction, "1000") + // Threshold to logging compaction trace, in seconds. CONF_mInt32(base_compaction_trace_threshold, "10"); CONF_mInt32(cumulative_compaction_trace_threshold, "2"); diff --git a/be/src/common/configbase.h b/be/src/common/configbase.h index 9dd58bb7b62071..2710ef7da62352 100644 --- a/be/src/common/configbase.h +++ b/be/src/common/configbase.h @@ -86,6 +86,7 @@ class Register { #define CONF_mInt32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, true) #define CONF_mInt64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, true) #define CONF_mDouble(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, true) +#define CONF_uInt32(name, defaultstr) DEFINE_FIELD(uint32_t, name, defaultstr, false) #else #define CONF_Bool(name, defaultstr) DECLARE_FIELD(bool, name) #define CONF_Int16(name, defaultstr) DECLARE_FIELD(int16_t, name) @@ -104,6 +105,7 @@ class Register { #define CONF_mInt32(name, defaultstr) DECLARE_FIELD(int32_t, name) #define CONF_mInt64(name, defaultstr) DECLARE_FIELD(int64_t, name) #define CONF_mDouble(name, defaultstr) DECLARE_FIELD(double, name) +#define CONF_uInt32(name, defaultstr) DECLARE_FIELD(uint32_t, name) #endif // configuration properties load from config file. diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt index 13c11a073c3075..3768b5eb8217aa 100644 --- a/be/src/olap/CMakeLists.txt +++ b/be/src/olap/CMakeLists.txt @@ -34,6 +34,7 @@ add_library(Olap STATIC bloom_filter_writer.cpp byte_buffer.cpp compaction.cpp + compaction_permit_limiter.cpp comparison_predicate.cpp compress.cpp cumulative_compaction.cpp diff --git a/be/src/olap/compaction_permit_limiter.cpp b/be/src/olap/compaction_permit_limiter.cpp new file mode 100644 index 00000000000000..a1c1e3b764db09 --- /dev/null +++ b/be/src/olap/compaction_permit_limiter.cpp @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/compaction_permit_limiter.h" + +namespace doris { + +uint32_t CompactionPermitLimiter::_total_permits; +uint32_t CompactionPermitLimiter::_used_permits; +bool CompactionPermitLimiter::_over_sold; +CountDownLatch CompactionPermitLimiter::_threads_latch; + +CompactionPermitLimiter::CompactionPermitLimiter() : _threads_latch(1) { }; + +OLAPStatus CompactionPermitLimiter::init(uint32_t total_permits, bool over_sold) { + _total_permits = total_permits; + _over_sold = over_sold; + return OLAP_SUCCESS; +} + +bool CompactionPermitLimiter::request(uint32_t permits) { + if (permits > _total_permits) { + if (_over_sold) { + // wait_until used_permits==0 + while(!_threads_latch.wait_for(MonoDelta::FromSeconds(5)) && _used_permits != 0); + _used_permits = _total_permits; + return true; + } else { + return false; + } + } else { + if ((_total_permits - _used_permits) < permits) { + return false; + } + _used_permits += permits; + return true; + } +} + +OLAPStatus CompactionPermitLimiter::release(uint32_t permits) { + if (permits > _total_permits) { + _used_permits = 0; + } else { + _used_permits = _used_permits - permits; + } + return OLAP_SUCCESS; +} + +} // namespace doris diff --git a/be/src/olap/compaction_permit_limiter.h b/be/src/olap/compaction_permit_limiter.h new file mode 100644 index 00000000000000..fecabc17be9db2 --- /dev/null +++ b/be/src/olap/compaction_permit_limiter.h @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_COMPACTION_PERMIT_LIMITER_H +#define DORIS_BE_SRC_OLAP_COMPACTION_PERMIT_LIMITER_H + +#include + +#include "olap/olap_common.h" +#include "olap/olap_define.h" +#include "olap/utils.h" +#include "util/countdown_latch.h" + +namespace doris { + +// This class is a base class for compaction management. +class CompactionPermitLimiter { +public: + CompactionPermitLimiter(); + virtual ~CompactionPermitLimiter() { }; + + static OLAPStatus init(uint32_t total_permits, bool _over_sold); + + static bool request(uint32_t permits); + + static OLAPStatus release(uint32_t permits); + + inline uint32_t total_permits() const; + inline void set_total_permits(uint32_t total_permits); + + inline uint32_t used_permits() const; + + inline bool is_over_sold() const; + inline void set_over_sold(bool over_sold); + +private: + static uint32_t _total_permits; + static uint32_t _used_permits; + static bool _over_sold; + + static CountDownLatch _threads_latch; +}; + +inline uint32_t CompactionPermitLimiter::total_permits() const { + return _total_permits; +} + +inline void CompactionPermitLimiter::set_total_permits(uint32_t total_permits) { + _total_permits = total_permits; +} + +inline uint32_t CompactionPermitLimiter::used_permits() const { + return _used_permits; +} + +inline bool CompactionPermitLimiter::is_over_sold() const { + return _over_sold; +} + +inline void CompactionPermitLimiter::set_over_sold(bool over_sold) { + _over_sold = over_sold; +} +} // namespace doris + +#endif // DORIS_BE_SRC_OLAP_COMPACTION_PERMIT_LIMITER_H diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 3f860f4f5971ec..65400f2ecec273 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -32,6 +32,7 @@ #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/storage_engine.h" +#include "olap/compaction_permit_limiter.h" #include "agent/cgroups_mgr.h" #include "util/time.h" @@ -87,7 +88,7 @@ Status StorageEngine::start_bg_threads() { max_compaction_concurrency = base_compaction_num_threads + cumulative_compaction_num_threads + 1; } Compaction::init(max_compaction_concurrency); - + /* _base_compaction_threads.reserve(base_compaction_num_threads); for (uint32_t i = 0; i < base_compaction_num_threads; ++i) { scoped_refptr base_compaction_thread; @@ -109,6 +110,16 @@ Status StorageEngine::start_bg_threads() { _cumulative_compaction_threads.emplace_back(cumulative_compaction_thread); } LOG(INFO) << "cumulative compaction threads started. number: " << cumulative_compaction_num_threads; + */ + + // compaction tasks producer thread + int32_t total_permits = config::total_permits_memory_for_compaction; + CompactionPermitLimiter::init(total_permits, true); + RETURN_IF_ERROR( + Thread::create("StorageEngine", "compaction_tasks_producer_thread", + [this]() { this->_compaction_tasks_producer_callback(); }, + &_compaction_tasks_producer_thread)); + LOG(INFO) << "compaction tasks producer thread started"; // tablet checkpoint thread for (auto data_dir : data_dirs) { @@ -169,6 +180,7 @@ void StorageEngine::_fd_cache_clean_callback() { } } +/* void StorageEngine::_base_compaction_thread_callback(DataDir* data_dir) { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); @@ -192,9 +204,22 @@ void StorageEngine::_base_compaction_thread_callback(DataDir* data_dir) { "force set to 1", interval); interval = 1; } - + } while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(interval))); } +*/ + +void StorageEngine::_base_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits) { +#ifdef GOOGLE_PROFILER + ProfilerRegisterThread(); +#endif + LOG(INFO) << "try to start base compaction process!"; + + _perform_base_compaction(tablet); + + CompactionPermitLimiter::release(permits); + _map_disk_compaction_num[tablet->data_dir()] = _map_disk_compaction_num[tablet->data_dir()] - 1; +} void StorageEngine::_garbage_sweeper_thread_callback() { #ifdef GOOGLE_PROFILER @@ -286,6 +311,7 @@ void StorageEngine::_check_cumulative_compaction_config() { } } +/* void StorageEngine::_cumulative_compaction_thread_callback(DataDir* data_dir) { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); @@ -312,6 +338,19 @@ void StorageEngine::_cumulative_compaction_thread_callback(DataDir* data_dir) { } } while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(interval))); } +*/ + +void StorageEngine::_cumulative_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits) { +#ifdef GOOGLE_PROFILER + ProfilerRegisterThread(); +#endif + LOG(INFO) << "try to start cumulative compaction process!"; + + _perform_cumulative_compaction(tablet); + + CompactionPermitLimiter::release(permits); + _map_disk_compaction_num[tablet->data_dir()] = _map_disk_compaction_num[tablet->data_dir()] - 1; +} void StorageEngine::_unused_rowset_monitor_thread_callback() { #ifdef GOOGLE_PROFILER @@ -391,4 +430,70 @@ void StorageEngine::_tablet_checkpoint_callback(DataDir* data_dir) { } while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(interval))); } +Status StorageEngine::_compaction_tasks_producer_callback() { +#ifdef GOOGLE_PROFILER + ProfilerRegisterThread(); +#endif + LOG(INFO) << "try to start compaction producer process!"; + + // convert store map to vector + std::vector data_dirs; + for (auto& tmp_store : _store_map) { + data_dirs.push_back(tmp_store.second); + _map_disk_compaction_num[tmp_store.second] = 0; + } + + int round = 0; + CompactionType compaction_type; + do { + if(round < 9) { + compaction_type = CompactionType::CUMULATIVE_COMPACTION; + round++; + } else { + compaction_type = CompactionType::BASE_COMPACTION; + round = 0; + } + vector tablets_compaction = _compaction_tasks_generator(compaction_type, data_dirs); + + for (int i = 0; i < tablets_compaction.size(); i++) { + uint32_t permits; + //estimate memory usage of compaction for tablets_compaction[i] + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + permits = tablets_compaction[i]->calc_cumulative_compaction_score(); + } else { + permits = tablets_compaction[i]->calc_base_compaction_score(); + } + if (CompactionPermitLimiter::request(permits)) { + // add task to thread pool + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + scoped_refptr cumulative_compaction_thread; + RETURN_IF_ERROR(Thread::create("StorageEngine", "cumulative_compaction_thread", + [this, i, tablets_compaction, permits]() { this->_cumulative_compaction_thread_callback(tablets_compaction[i], permits); }, + &cumulative_compaction_thread)); + } else { + scoped_refptr base_compaction_thread; + RETURN_IF_ERROR(Thread::create("StorageEngine", "base_compaction_thread", + [this, i, tablets_compaction, permits]() { this->_base_compaction_thread_callback(tablets_compaction[i], permits); }, + &base_compaction_thread)); + } + _map_disk_compaction_num[tablets_compaction[i]->data_dir()] = _map_disk_compaction_num[tablets_compaction[i]->data_dir()] + 1; + } else { + continue; + } + } + + } while (true); +} + +vector StorageEngine::_compaction_tasks_generator(CompactionType compaction_type, std::vector data_dirs) { + vector tablets_compaction; + std::random_shuffle(data_dirs.begin(), data_dirs.end()); + for (auto data_dir : data_dirs) { + if (!data_dir->reach_capacity_limit(0)) { + TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction(compaction_type, data_dir); + tablets_compaction.emplace_back(tablet); + } + } + return tablets_compaction; +} } // namespace doris diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index a07d6b9eb93558..968ef16aeac1e3 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -578,7 +578,7 @@ void StorageEngine::_start_clean_fd_cache() { VLOG(10) << "end clean file descritpor cache"; } -void StorageEngine::_perform_cumulative_compaction(DataDir* data_dir) { +void StorageEngine::_perform_cumulative_compaction(TabletSharedPtr best_tablet) { scoped_refptr trace(new Trace); MonotonicStopWatch watch; watch.start(); @@ -589,13 +589,14 @@ void StorageEngine::_perform_cumulative_compaction(DataDir* data_dir) { }); ADOPT_TRACE(trace.get()); TRACE("start to perform cumulative compaction"); + /* TabletSharedPtr best_tablet = _tablet_manager->find_best_tablet_to_compaction( CompactionType::CUMULATIVE_COMPACTION, data_dir); if (best_tablet == nullptr) { return; } TRACE("found best tablet $0", best_tablet->get_tablet_info().tablet_id); - + */ DorisMetrics::instance()->cumulative_compaction_request_total->increment(1); std::string tracker_label = "cumulative compaction " + std::to_string(syscall(__NR_gettid)); @@ -614,7 +615,7 @@ void StorageEngine::_perform_cumulative_compaction(DataDir* data_dir) { best_tablet->set_last_cumu_compaction_failure_time(0); } -void StorageEngine::_perform_base_compaction(DataDir* data_dir) { +void StorageEngine::_perform_base_compaction(TabletSharedPtr best_tablet) { scoped_refptr trace(new Trace); MonotonicStopWatch watch; watch.start(); @@ -625,13 +626,14 @@ void StorageEngine::_perform_base_compaction(DataDir* data_dir) { }); ADOPT_TRACE(trace.get()); TRACE("start to perform base compaction"); + /* TabletSharedPtr best_tablet = _tablet_manager->find_best_tablet_to_compaction( CompactionType::BASE_COMPACTION, data_dir); if (best_tablet == nullptr) { return; } TRACE("found best tablet $0", best_tablet->get_tablet_info().tablet_id); - + */ DorisMetrics::instance()->base_compaction_request_total->increment(1); std::string tracker_label = "base compaction " + std::to_string(syscall(__NR_gettid)); diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 0b3aacfee3e81e..1a14ef9e34c5ab 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -43,6 +43,7 @@ #include "olap/olap_meta.h" #include "olap/options.h" #include "olap/tablet_manager.h" +#include "olap/compaction_permit_limiter.h" #include "olap/tablet_sync_service.h" #include "olap/txn_manager.h" #include "olap/task/engine_task.h" @@ -212,11 +213,11 @@ class StorageEngine { void _unused_rowset_monitor_thread_callback(); // base compaction thread process function - void _base_compaction_thread_callback(DataDir* data_dir); + void _base_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permit); // check cumulative compaction config void _check_cumulative_compaction_config(); // cumulative process function - void _cumulative_compaction_thread_callback(DataDir* data_dir); + void _cumulative_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits); // garbage sweep thread process function. clear snapshot and trash folder void _garbage_sweeper_thread_callback(); @@ -238,8 +239,8 @@ class StorageEngine { void _parse_default_rowset_type(); void _start_clean_fd_cache(); - void _perform_cumulative_compaction(DataDir* data_dir); - void _perform_base_compaction(DataDir* data_dir); + void _perform_cumulative_compaction(TabletSharedPtr best_tablet); + void _perform_base_compaction(TabletSharedPtr best_tablet); // 清理trash和snapshot文件，返回清理后的磁盘使用量 OLAPStatus _start_trash_sweep(double *usage); // 磁盘状态监测。监测unused_flag路劲新的对应root_path unused标识位， @@ -248,6 +249,9 @@ class StorageEngine { // 重新加载数据。 void _start_disk_stat_monitor(); + Status _compaction_tasks_producer_callback(); + vector _compaction_tasks_generator(CompactionType compaction_type, std::vector data_dirs); + private: struct CompactionCandidate { CompactionCandidate(uint32_t nicumulative_compaction_, int64_t tablet_id_, uint32_t index_) : @@ -313,6 +317,7 @@ class StorageEngine { std::vector> _base_compaction_threads; // threads to check cumulative std::vector> _cumulative_compaction_threads; + scoped_refptr _compaction_tasks_producer_thread; scoped_refptr _fd_cache_clean_thread; // threads to clean all file descriptor not actively in use std::vector> _path_gc_threads; @@ -343,6 +348,8 @@ class StorageEngine { HeartbeatFlags* _heartbeat_flags; DISALLOW_COPY_AND_ASSIGN(StorageEngine); + + std::map _map_disk_compaction_num; }; } // namespace doris From d96e0ee99022f00e595aef460f7a87b780b71675 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Sun, 20 Sep 2020 20:53:58 +0800 Subject: [PATCH 02/27] compaction memory optimize and code refactor --- be/src/olap/compaction_permit_limiter.cpp | 9 ++++----- be/src/olap/compaction_permit_limiter.h | 5 +---- be/src/olap/olap_server.cpp | 21 ++++++++++++++++++--- be/src/olap/storage_engine.h | 9 +++++++-- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/be/src/olap/compaction_permit_limiter.cpp b/be/src/olap/compaction_permit_limiter.cpp index a1c1e3b764db09..bbd1c823ffd358 100644 --- a/be/src/olap/compaction_permit_limiter.cpp +++ b/be/src/olap/compaction_permit_limiter.cpp @@ -22,13 +22,11 @@ namespace doris { uint32_t CompactionPermitLimiter::_total_permits; uint32_t CompactionPermitLimiter::_used_permits; bool CompactionPermitLimiter::_over_sold; -CountDownLatch CompactionPermitLimiter::_threads_latch; - -CompactionPermitLimiter::CompactionPermitLimiter() : _threads_latch(1) { }; OLAPStatus CompactionPermitLimiter::init(uint32_t total_permits, bool over_sold) { _total_permits = total_permits; _over_sold = over_sold; + _used_permits = 0; return OLAP_SUCCESS; } @@ -36,7 +34,9 @@ bool CompactionPermitLimiter::request(uint32_t permits) { if (permits > _total_permits) { if (_over_sold) { // wait_until used_permits==0 - while(!_threads_latch.wait_for(MonoDelta::FromSeconds(5)) && _used_permits != 0); + while(_used_permits != 0) { + sleep(5); + } _used_permits = _total_permits; return true; } else { @@ -59,5 +59,4 @@ OLAPStatus CompactionPermitLimiter::release(uint32_t permits) { } return OLAP_SUCCESS; } - } // namespace doris diff --git a/be/src/olap/compaction_permit_limiter.h b/be/src/olap/compaction_permit_limiter.h index fecabc17be9db2..b5e637a37a2729 100644 --- a/be/src/olap/compaction_permit_limiter.h +++ b/be/src/olap/compaction_permit_limiter.h @@ -23,14 +23,13 @@ #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/utils.h" -#include "util/countdown_latch.h" namespace doris { // This class is a base class for compaction management. class CompactionPermitLimiter { public: - CompactionPermitLimiter(); + CompactionPermitLimiter() {}; virtual ~CompactionPermitLimiter() { }; static OLAPStatus init(uint32_t total_permits, bool _over_sold); @@ -51,8 +50,6 @@ class CompactionPermitLimiter { static uint32_t _total_permits; static uint32_t _used_permits; static bool _over_sold; - - static CountDownLatch _threads_latch; }; inline uint32_t CompactionPermitLimiter::total_permits() const { diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 65400f2ecec273..61857d4addd944 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -33,7 +33,7 @@ #include "olap/olap_define.h" #include "olap/storage_engine.h" #include "olap/compaction_permit_limiter.h" -#include "agent/cgroups_mgr.h" +//#include "agent/cgroups_mgr.h" #include "util/time.h" using std::string; @@ -209,12 +209,14 @@ void StorageEngine::_base_compaction_thread_callback(DataDir* data_dir) { } */ -void StorageEngine::_base_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits) { +void StorageEngine::_base_compaction_thread_callback() { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif LOG(INFO) << "try to start base compaction process!"; + TabletSharedPtr tablet = _tablet; + uint32_t permits = _permits; _perform_base_compaction(tablet); CompactionPermitLimiter::release(permits); @@ -340,12 +342,15 @@ void StorageEngine::_cumulative_compaction_thread_callback(DataDir* data_dir) { } */ -void StorageEngine::_cumulative_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits) { +void StorageEngine::_cumulative_compaction_thread_callback() { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif LOG(INFO) << "try to start cumulative compaction process!"; + TabletSharedPtr tablet = _tablet; + uint32_t permits = _permits; + _perform_cumulative_compaction(tablet); CompactionPermitLimiter::release(permits); @@ -436,6 +441,9 @@ Status StorageEngine::_compaction_tasks_producer_callback() { #endif LOG(INFO) << "try to start compaction producer process!"; + int thread_num = config::max_compaction_concurrency; + ThreadPoolBuilder("CompactionTaskThreadPool").set_min_threads(thread_num).set_max_threads(thread_num).build(&_thread_pool); + // convert store map to vector std::vector data_dirs; for (auto& tmp_store : _store_map) { @@ -463,18 +471,25 @@ Status StorageEngine::_compaction_tasks_producer_callback() { } else { permits = tablets_compaction[i]->calc_base_compaction_score(); } + _tablet = tablets_compaction[i]; + _permits = permits; if (CompactionPermitLimiter::request(permits)) { // add task to thread pool if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + _thread_pool->submit_func(std::bind(&StorageEngine::_cumulative_compaction_thread_callback, this)); + /* scoped_refptr cumulative_compaction_thread; RETURN_IF_ERROR(Thread::create("StorageEngine", "cumulative_compaction_thread", [this, i, tablets_compaction, permits]() { this->_cumulative_compaction_thread_callback(tablets_compaction[i], permits); }, &cumulative_compaction_thread)); + */ } else { + _thread_pool->submit_func(std::bind(&StorageEngine::_base_compaction_thread_callback, this)); /* scoped_refptr base_compaction_thread; RETURN_IF_ERROR(Thread::create("StorageEngine", "base_compaction_thread", [this, i, tablets_compaction, permits]() { this->_base_compaction_thread_callback(tablets_compaction[i], permits); }, &base_compaction_thread)); + */ } _map_disk_compaction_num[tablets_compaction[i]->data_dir()] = _map_disk_compaction_num[tablets_compaction[i]->data_dir()] + 1; } else { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 1a14ef9e34c5ab..47536da9e0f18d 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -52,6 +52,8 @@ #include "runtime/heartbeat_flags.h" #include "util/countdown_latch.h" #include "util/thread.h" +#include "util/threadpool.h" + namespace doris { @@ -213,11 +215,11 @@ class StorageEngine { void _unused_rowset_monitor_thread_callback(); // base compaction thread process function - void _base_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permit); + void _base_compaction_thread_callback(); // check cumulative compaction config void _check_cumulative_compaction_config(); // cumulative process function - void _cumulative_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits); + void _cumulative_compaction_thread_callback(); // garbage sweep thread process function. clear snapshot and trash folder void _garbage_sweeper_thread_callback(); @@ -350,6 +352,9 @@ class StorageEngine { DISALLOW_COPY_AND_ASSIGN(StorageEngine); std::map _map_disk_compaction_num; + std::unique_ptr _thread_pool; + TabletSharedPtr _tablet; + uint32_t _permits; }; } // namespace doris From 908788aab68dbdfd4f98698bfcef1b633207c40d Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Sun, 20 Sep 2020 21:03:16 +0800 Subject: [PATCH 03/27] compaction memory optimize and code refactor --- be/src/olap/olap_server.cpp | 96 +------------------------------------ 1 file changed, 2 insertions(+), 94 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 61857d4addd944..d5b2561e9d1031 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -88,29 +88,6 @@ Status StorageEngine::start_bg_threads() { max_compaction_concurrency = base_compaction_num_threads + cumulative_compaction_num_threads + 1; } Compaction::init(max_compaction_concurrency); - /* - _base_compaction_threads.reserve(base_compaction_num_threads); - for (uint32_t i = 0; i < base_compaction_num_threads; ++i) { - scoped_refptr base_compaction_thread; - RETURN_IF_ERROR( - Thread::create("StorageEngine", "base_compaction_thread", - [this, i, data_dir_num, data_dirs]() { this->_base_compaction_thread_callback(data_dirs[i % data_dir_num]); }, - &base_compaction_thread)); - _base_compaction_threads.emplace_back(base_compaction_thread); - } - LOG(INFO) << "base compaction threads started. number: " << base_compaction_num_threads; - - _cumulative_compaction_threads.reserve(cumulative_compaction_num_threads); - for (uint32_t i = 0; i < cumulative_compaction_num_threads; ++i) { - scoped_refptr cumulative_compaction_thread; - RETURN_IF_ERROR( - Thread::create("StorageEngine", "cumulative_compaction_thread", - [this, i, data_dir_num, data_dirs]() { this->_cumulative_compaction_thread_callback(data_dirs[i % data_dir_num]); }, - &cumulative_compaction_thread)); - _cumulative_compaction_threads.emplace_back(cumulative_compaction_thread); - } - LOG(INFO) << "cumulative compaction threads started. number: " << cumulative_compaction_num_threads; - */ // compaction tasks producer thread int32_t total_permits = config::total_permits_memory_for_compaction; @@ -180,35 +157,6 @@ void StorageEngine::_fd_cache_clean_callback() { } } -/* -void StorageEngine::_base_compaction_thread_callback(DataDir* data_dir) { -#ifdef GOOGLE_PROFILER - ProfilerRegisterThread(); -#endif - - int32_t interval = config::base_compaction_check_interval_seconds; - do { - if (!config::disable_auto_compaction) { - // must be here, because this thread is start on start and - // cgroup is not initialized at this time - // add tid to cgroup - CgroupsMgr::apply_system_cgroup(); - if (!data_dir->reach_capacity_limit(0)) { - _perform_base_compaction(data_dir); - } - } - - interval = config::base_compaction_check_interval_seconds; - if (interval <= 0) { - OLAP_LOG_WARNING("base compaction check interval config is illegal: [%d], " - "force set to 1", interval); - interval = 1; - } - - } while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(interval))); -} -*/ - void StorageEngine::_base_compaction_thread_callback() { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); @@ -313,35 +261,6 @@ void StorageEngine::_check_cumulative_compaction_config() { } } -/* -void StorageEngine::_cumulative_compaction_thread_callback(DataDir* data_dir) { -#ifdef GOOGLE_PROFILER - ProfilerRegisterThread(); -#endif - LOG(INFO) << "try to start cumulative compaction process!"; - - int32_t interval = config::cumulative_compaction_check_interval_seconds; - do { - if (!config::disable_auto_compaction) { - // must be here, because this thread is start on start and - // cgroup is not initialized at this time - // add tid to cgroup - CgroupsMgr::apply_system_cgroup(); - if (!data_dir->reach_capacity_limit(0)) { - _perform_cumulative_compaction(data_dir); - } - } - - interval = config::cumulative_compaction_check_interval_seconds; - if (interval <= 0) { - LOG(WARNING) << "cumulative compaction check interval config is illegal:" << interval - << "will be forced set to one"; - interval = 1; - } - } while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(interval))); -} -*/ - void StorageEngine::_cumulative_compaction_thread_callback() { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); @@ -477,26 +396,15 @@ Status StorageEngine::_compaction_tasks_producer_callback() { // add task to thread pool if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { _thread_pool->submit_func(std::bind(&StorageEngine::_cumulative_compaction_thread_callback, this)); - /* - scoped_refptr cumulative_compaction_thread; - RETURN_IF_ERROR(Thread::create("StorageEngine", "cumulative_compaction_thread", - [this, i, tablets_compaction, permits]() { this->_cumulative_compaction_thread_callback(tablets_compaction[i], permits); }, - &cumulative_compaction_thread)); - */ } else { - _thread_pool->submit_func(std::bind(&StorageEngine::_base_compaction_thread_callback, this)); /* - scoped_refptr base_compaction_thread; - RETURN_IF_ERROR(Thread::create("StorageEngine", "base_compaction_thread", - [this, i, tablets_compaction, permits]() { this->_base_compaction_thread_callback(tablets_compaction[i], permits); }, - &base_compaction_thread)); - */ + _thread_pool->submit_func(std::bind(&StorageEngine::_base_compaction_thread_callback, this)); } _map_disk_compaction_num[tablets_compaction[i]->data_dir()] = _map_disk_compaction_num[tablets_compaction[i]->data_dir()] + 1; } else { continue; } } - + sleep(5); } while (true); } From 4bea5d544020c7121be2e8523393714e204623a7 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Sun, 20 Sep 2020 21:04:36 +0800 Subject: [PATCH 04/27] compaction memory optimize and code refactor --- be/src/olap/olap_server.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index d5b2561e9d1031..1cce87d0cc4c03 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -33,7 +33,6 @@ #include "olap/olap_define.h" #include "olap/storage_engine.h" #include "olap/compaction_permit_limiter.h" -//#include "agent/cgroups_mgr.h" #include "util/time.h" using std::string; From 72e4e7973aee215fcca0b223305e335d3e4ec83f Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Sun, 20 Sep 2020 21:11:48 +0800 Subject: [PATCH 05/27] compaction memory optimize and code refactor --- be/src/olap/storage_engine.cpp | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 968ef16aeac1e3..b5d189ebbdf414 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -589,14 +589,7 @@ void StorageEngine::_perform_cumulative_compaction(TabletSharedPtr best_tablet) }); ADOPT_TRACE(trace.get()); TRACE("start to perform cumulative compaction"); - /* - TabletSharedPtr best_tablet = _tablet_manager->find_best_tablet_to_compaction( - CompactionType::CUMULATIVE_COMPACTION, data_dir); - if (best_tablet == nullptr) { - return; - } - TRACE("found best tablet $0", best_tablet->get_tablet_info().tablet_id); - */ + DorisMetrics::instance()->cumulative_compaction_request_total->increment(1); std::string tracker_label = "cumulative compaction " + std::to_string(syscall(__NR_gettid)); @@ -626,14 +619,7 @@ void StorageEngine::_perform_base_compaction(TabletSharedPtr best_tablet) { }); ADOPT_TRACE(trace.get()); TRACE("start to perform base compaction"); - /* - TabletSharedPtr best_tablet = _tablet_manager->find_best_tablet_to_compaction( - CompactionType::BASE_COMPACTION, data_dir); - if (best_tablet == nullptr) { - return; - } - TRACE("found best tablet $0", best_tablet->get_tablet_info().tablet_id); - */ + DorisMetrics::instance()->base_compaction_request_total->increment(1); std::string tracker_label = "base compaction " + std::to_string(syscall(__NR_gettid)); From 506afdb34de9ba61a1422ca0f52c7ae9681ff259 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Mon, 21 Sep 2020 14:45:41 +0800 Subject: [PATCH 06/27] add compaction status for tablet --- be/src/olap/olap_server.cpp | 7 ++++--- be/src/olap/tablet.h | 5 +++++ be/src/olap/tablet_manager.cpp | 3 ++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 1cce87d0cc4c03..65097a975a73c0 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -164,8 +164,9 @@ void StorageEngine::_base_compaction_thread_callback() { TabletSharedPtr tablet = _tablet; uint32_t permits = _permits; + tablet->set_compaction_now(true); _perform_base_compaction(tablet); - + tablet->set_compaction_now(false); CompactionPermitLimiter::release(permits); _map_disk_compaction_num[tablet->data_dir()] = _map_disk_compaction_num[tablet->data_dir()] - 1; } @@ -268,9 +269,9 @@ void StorageEngine::_cumulative_compaction_thread_callback() { TabletSharedPtr tablet = _tablet; uint32_t permits = _permits; - + tablet->set_compaction_now(true); _perform_cumulative_compaction(tablet); - + tablet->set_compaction_now(false); CompactionPermitLimiter::release(permits); _map_disk_compaction_num[tablet->data_dir()] = _map_disk_compaction_num[tablet->data_dir()] - 1; } diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index c32daad3b51008..540d853e36f246 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -233,6 +233,9 @@ class Tablet : public BaseTablet { // return a json string to show the compaction status of this tablet void get_compaction_status(std::string* json_result); + inline void set_compaction_now(bool compaction_now) { _compaction_now = compaction_now; } + inline bool compaction_now() { return _compaction_now; } + private: OLAPStatus _init_once_action(); void _print_missed_versions(const std::vector& missed_versions) const; @@ -302,6 +305,8 @@ class Tablet : public BaseTablet { std::string _cumulative_compaction_type; DISALLOW_COPY_AND_ASSIGN(Tablet); + bool _compaction_now; + public: IntCounter* flush_bytes; IntCounter* flush_count; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 6260f2163184e3..020d088fdbba94 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -710,7 +710,8 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(CompactionType com if (tablet_ptr->data_dir()->path_hash() != data_dir->path_hash() || !tablet_ptr->is_used() || !tablet_ptr->init_succeeded() - || !tablet_ptr->can_do_compaction()) { + || !tablet_ptr->can_do_compaction() + || tablet_ptr->compaction_now()) { continue; } From dce66cde3ab3a22640c5f2464164b88705482ea7 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Mon, 21 Sep 2020 16:00:25 +0800 Subject: [PATCH 07/27] add lambda function for submit callback --- be/src/olap/olap_server.cpp | 18 +++++++----------- be/src/olap/storage_engine.h | 6 ++---- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 65097a975a73c0..d01c27c9c8af89 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -89,7 +89,7 @@ Status StorageEngine::start_bg_threads() { Compaction::init(max_compaction_concurrency); // compaction tasks producer thread - int32_t total_permits = config::total_permits_memory_for_compaction; + uint32_t total_permits = config::total_permits_memory_for_compaction; CompactionPermitLimiter::init(total_permits, true); RETURN_IF_ERROR( Thread::create("StorageEngine", "compaction_tasks_producer_thread", @@ -156,14 +156,12 @@ void StorageEngine::_fd_cache_clean_callback() { } } -void StorageEngine::_base_compaction_thread_callback() { +void StorageEngine::_base_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits) { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif LOG(INFO) << "try to start base compaction process!"; - TabletSharedPtr tablet = _tablet; - uint32_t permits = _permits; tablet->set_compaction_now(true); _perform_base_compaction(tablet); tablet->set_compaction_now(false); @@ -261,14 +259,12 @@ void StorageEngine::_check_cumulative_compaction_config() { } } -void StorageEngine::_cumulative_compaction_thread_callback() { +void StorageEngine::_cumulative_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits) { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif LOG(INFO) << "try to start cumulative compaction process!"; - TabletSharedPtr tablet = _tablet; - uint32_t permits = _permits; tablet->set_compaction_now(true); _perform_cumulative_compaction(tablet); tablet->set_compaction_now(false); @@ -390,14 +386,14 @@ Status StorageEngine::_compaction_tasks_producer_callback() { } else { permits = tablets_compaction[i]->calc_base_compaction_score(); } - _tablet = tablets_compaction[i]; - _permits = permits; if (CompactionPermitLimiter::request(permits)) { // add task to thread pool if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - _thread_pool->submit_func(std::bind(&StorageEngine::_cumulative_compaction_thread_callback, this)); + //_thread_pool->submit_func(std::bind(&StorageEngine::_cumulative_compaction_thread_callback, tablets_compaction[i], permits)); + _thread_pool->submit_func([this, i, tablets_compaction, permits]() {this->_cumulative_compaction_thread_callback(tablets_compaction[i], permits);}); } else { - _thread_pool->submit_func(std::bind(&StorageEngine::_base_compaction_thread_callback, this)); + //_thread_pool->submit_func(std::bind(&StorageEngine::_base_compaction_thread_callback, tablets_compaction[i], permits)); + _thread_pool->submit_func([this, i, tablets_compaction, permits]() {this->_base_compaction_thread_callback(tablets_compaction[i], permits);}); } _map_disk_compaction_num[tablets_compaction[i]->data_dir()] = _map_disk_compaction_num[tablets_compaction[i]->data_dir()] + 1; } else { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 47536da9e0f18d..5ab6bf78baf419 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -215,11 +215,11 @@ class StorageEngine { void _unused_rowset_monitor_thread_callback(); // base compaction thread process function - void _base_compaction_thread_callback(); + void _base_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits); // check cumulative compaction config void _check_cumulative_compaction_config(); // cumulative process function - void _cumulative_compaction_thread_callback(); + void _cumulative_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits); // garbage sweep thread process function. clear snapshot and trash folder void _garbage_sweeper_thread_callback(); @@ -353,8 +353,6 @@ class StorageEngine { std::map _map_disk_compaction_num; std::unique_ptr _thread_pool; - TabletSharedPtr _tablet; - uint32_t _permits; }; } // namespace doris From ab4722a952caadcd3293ad7ddd6506b5fde2a897 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Mon, 21 Sep 2020 16:48:12 +0800 Subject: [PATCH 08/27] code refactor --- be/src/olap/olap_server.cpp | 34 ++++++++++++++-------------------- be/src/olap/tablet.h | 5 ----- be/src/olap/tablet_manager.cpp | 3 +-- 3 files changed, 15 insertions(+), 27 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index d01c27c9c8af89..9c3eb8e9d9e80b 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -162,9 +162,7 @@ void StorageEngine::_base_compaction_thread_callback(TabletSharedPtr tablet, uin #endif LOG(INFO) << "try to start base compaction process!"; - tablet->set_compaction_now(true); _perform_base_compaction(tablet); - tablet->set_compaction_now(false); CompactionPermitLimiter::release(permits); _map_disk_compaction_num[tablet->data_dir()] = _map_disk_compaction_num[tablet->data_dir()] - 1; } @@ -265,9 +263,7 @@ void StorageEngine::_cumulative_compaction_thread_callback(TabletSharedPtr table #endif LOG(INFO) << "try to start cumulative compaction process!"; - tablet->set_compaction_now(true); _perform_cumulative_compaction(tablet); - tablet->set_compaction_now(false); CompactionPermitLimiter::release(permits); _map_disk_compaction_num[tablet->data_dir()] = _map_disk_compaction_num[tablet->data_dir()] - 1; } @@ -379,25 +375,23 @@ Status StorageEngine::_compaction_tasks_producer_callback() { vector tablets_compaction = _compaction_tasks_generator(compaction_type, data_dirs); for (int i = 0; i < tablets_compaction.size(); i++) { - uint32_t permits; - //estimate memory usage of compaction for tablets_compaction[i] - if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - permits = tablets_compaction[i]->calc_cumulative_compaction_score(); - } else { - permits = tablets_compaction[i]->calc_base_compaction_score(); - } - if (CompactionPermitLimiter::request(permits)) { - // add task to thread pool + if(_map_disk_compaction_num[tablets_compaction[i]->data_dir()] == 0) { + uint32_t permits; if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - //_thread_pool->submit_func(std::bind(&StorageEngine::_cumulative_compaction_thread_callback, tablets_compaction[i], permits)); - _thread_pool->submit_func([this, i, tablets_compaction, permits]() {this->_cumulative_compaction_thread_callback(tablets_compaction[i], permits);}); + permits = tablets_compaction[i]->calc_cumulative_compaction_score(); + } else { + permits = tablets_compaction[i]->calc_base_compaction_score(); + } + if (CompactionPermitLimiter::request(permits)) { + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + _thread_pool->submit_func([this, i, tablets_compaction, permits]() {this->_cumulative_compaction_thread_callback(tablets_compaction[i], permits);}); + } else { + _thread_pool->submit_func([this, i, tablets_compaction, permits]() {this->_base_compaction_thread_callback(tablets_compaction[i], permits);}); + } + _map_disk_compaction_num[tablets_compaction[i]->data_dir()] = _map_disk_compaction_num[tablets_compaction[i]->data_dir()] + 1; } else { - //_thread_pool->submit_func(std::bind(&StorageEngine::_base_compaction_thread_callback, tablets_compaction[i], permits)); - _thread_pool->submit_func([this, i, tablets_compaction, permits]() {this->_base_compaction_thread_callback(tablets_compaction[i], permits);}); + continue; } - _map_disk_compaction_num[tablets_compaction[i]->data_dir()] = _map_disk_compaction_num[tablets_compaction[i]->data_dir()] + 1; - } else { - continue; } } sleep(5); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 540d853e36f246..c32daad3b51008 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -233,9 +233,6 @@ class Tablet : public BaseTablet { // return a json string to show the compaction status of this tablet void get_compaction_status(std::string* json_result); - inline void set_compaction_now(bool compaction_now) { _compaction_now = compaction_now; } - inline bool compaction_now() { return _compaction_now; } - private: OLAPStatus _init_once_action(); void _print_missed_versions(const std::vector& missed_versions) const; @@ -305,8 +302,6 @@ class Tablet : public BaseTablet { std::string _cumulative_compaction_type; DISALLOW_COPY_AND_ASSIGN(Tablet); - bool _compaction_now; - public: IntCounter* flush_bytes; IntCounter* flush_count; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 020d088fdbba94..6260f2163184e3 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -710,8 +710,7 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(CompactionType com if (tablet_ptr->data_dir()->path_hash() != data_dir->path_hash() || !tablet_ptr->is_used() || !tablet_ptr->init_succeeded() - || !tablet_ptr->can_do_compaction() - || tablet_ptr->compaction_now()) { + || !tablet_ptr->can_do_compaction()) { continue; } From 71cffec1357f889df89743cad93de256f2aa7ca4 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Mon, 21 Sep 2020 17:17:59 +0800 Subject: [PATCH 09/27] code refactor --- be/src/olap/olap_server.cpp | 6 +++--- be/src/olap/storage_engine.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 9c3eb8e9d9e80b..2512fcd9fbf1e3 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -353,7 +353,7 @@ Status StorageEngine::_compaction_tasks_producer_callback() { LOG(INFO) << "try to start compaction producer process!"; int thread_num = config::max_compaction_concurrency; - ThreadPoolBuilder("CompactionTaskThreadPool").set_min_threads(thread_num).set_max_threads(thread_num).build(&_thread_pool); + ThreadPoolBuilder("CompactionTaskThreadPool").set_min_threads(thread_num).set_max_threads(thread_num).build(&_thread_pool_compaction); // convert store map to vector std::vector data_dirs; @@ -384,9 +384,9 @@ Status StorageEngine::_compaction_tasks_producer_callback() { } if (CompactionPermitLimiter::request(permits)) { if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - _thread_pool->submit_func([this, i, tablets_compaction, permits]() {this->_cumulative_compaction_thread_callback(tablets_compaction[i], permits);}); + _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_cumulative_compaction_thread_callback(tablets_compaction[i], permits);}); } else { - _thread_pool->submit_func([this, i, tablets_compaction, permits]() {this->_base_compaction_thread_callback(tablets_compaction[i], permits);}); + _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_base_compaction_thread_callback(tablets_compaction[i], permits);}); } _map_disk_compaction_num[tablets_compaction[i]->data_dir()] = _map_disk_compaction_num[tablets_compaction[i]->data_dir()] + 1; } else { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 5ab6bf78baf419..0d3f8f1b00a00e 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -352,7 +352,7 @@ class StorageEngine { DISALLOW_COPY_AND_ASSIGN(StorageEngine); std::map _map_disk_compaction_num; - std::unique_ptr _thread_pool; + std::unique_ptr _thread_pool_compaction; }; } // namespace doris From 64681973707871fa500f1dd889f800094f1f9e78 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Mon, 21 Sep 2020 19:32:25 +0800 Subject: [PATCH 10/27] code refactor --- be/src/common/config.h | 18 ++++++-------- be/src/common/configbase.h | 2 -- be/src/olap/compaction.cpp | 9 ------- be/src/olap/compaction.h | 5 ---- be/src/olap/compaction_permit_limiter.cpp | 5 ---- be/src/olap/compaction_permit_limiter.h | 12 +++++----- be/src/olap/olap_server.cpp | 29 ++++++----------------- be/src/olap/storage_engine.cpp | 1 + be/src/olap/storage_engine.h | 4 +++- 9 files changed, 24 insertions(+), 61 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index aa2fe41b34e34a..b3edbff5ee085d 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -269,7 +269,6 @@ namespace config { // CONF_Int64(base_compaction_end_hour, "7"); CONF_mInt32(base_compaction_check_interval_seconds, "60"); CONF_mInt64(base_compaction_num_cumulative_deltas, "5"); - CONF_Int32(base_compaction_num_threads_per_disk, "1"); CONF_mDouble(base_cumulative_delta_ratio, "0.3"); CONF_mInt64(base_compaction_interval_seconds_since_last_operation, "86400"); CONF_mInt32(base_compaction_write_mbytes_per_sec, "5"); @@ -299,7 +298,6 @@ namespace config { CONF_mInt32(cumulative_compaction_check_interval_seconds, "10"); CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5"); CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000"); - CONF_Int32(cumulative_compaction_num_threads_per_disk, "1"); CONF_mInt64(cumulative_compaction_budgeted_bytes, "104857600"); // CONF_Int32(cumulative_compaction_write_mbytes_per_sec, "100"); // cumulative compaction skips recently published deltas in order to prevent @@ -310,15 +308,13 @@ namespace config { // if compaction of a tablet failed, this tablet should not be chosen to // compaction until this interval passes. CONF_mInt64(min_compaction_failure_interval_sec, "600"); // 10 min - // Too many compaction tasks may run out of memory. - // This config is to limit the max concurrency of running compaction tasks. - // -1 means no limit, and the max concurrency will be: - // C = (cumulative_compaction_num_threads_per_disk + base_compaction_num_threads_per_disk) * dir_num - // set it to larger than C will be set to equal to C. - // This config can be set to 0, which means to forbid any compaction, for some special cases. - CONF_Int32(max_compaction_concurrency, "-1"); - - CONF_uInt32(total_permits_memory_for_compaction, "1000") + + // This config can be set to limit thread number in compaction thread pool. + CONF_mInt32(min_compaction_threads, "10"); + CONF_mInt32(max_compaction_threads, "10"); + + // This config can be set to limit memory for compaction. + CONF_mInt64(total_permits_memory_for_compaction, "10000") // Threshold to logging compaction trace, in seconds. CONF_mInt32(base_compaction_trace_threshold, "10"); diff --git a/be/src/common/configbase.h b/be/src/common/configbase.h index 2710ef7da62352..9dd58bb7b62071 100644 --- a/be/src/common/configbase.h +++ b/be/src/common/configbase.h @@ -86,7 +86,6 @@ class Register { #define CONF_mInt32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, true) #define CONF_mInt64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, true) #define CONF_mDouble(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, true) -#define CONF_uInt32(name, defaultstr) DEFINE_FIELD(uint32_t, name, defaultstr, false) #else #define CONF_Bool(name, defaultstr) DECLARE_FIELD(bool, name) #define CONF_Int16(name, defaultstr) DECLARE_FIELD(int16_t, name) @@ -105,7 +104,6 @@ class Register { #define CONF_mInt32(name, defaultstr) DECLARE_FIELD(int32_t, name) #define CONF_mInt64(name, defaultstr) DECLARE_FIELD(int64_t, name) #define CONF_mDouble(name, defaultstr) DECLARE_FIELD(double, name) -#define CONF_uInt32(name, defaultstr) DECLARE_FIELD(uint32_t, name) #endif // configuration properties load from config file. diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 67b56f6e763172..d0e5bc88da1383 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -25,8 +25,6 @@ using std::vector; namespace doris { -Semaphore Compaction::_concurrency_sem; - Compaction::Compaction(TabletSharedPtr tablet, const std::string& label, const std::shared_ptr& parent_tracker) : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker)), _readers_tracker(MemTracker::CreateTracker(-1, "readers tracker", _mem_tracker)), @@ -37,16 +35,9 @@ Compaction::Compaction(TabletSharedPtr tablet, const std::string& label, const s Compaction::~Compaction() {} -OLAPStatus Compaction::init(int concurreny) { - _concurrency_sem.set_count(concurreny); - return OLAP_SUCCESS; -} - OLAPStatus Compaction::do_compaction() { - _concurrency_sem.wait(); TRACE("got concurrency lock and start to do compaction"); OLAPStatus st = do_compaction_impl(); - _concurrency_sem.signal(); return st; } diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h index 6c4b438aaeb260..45735937f5adc2 100644 --- a/be/src/olap/compaction.h +++ b/be/src/olap/compaction.h @@ -49,8 +49,6 @@ class Compaction { virtual OLAPStatus compact() = 0; - static OLAPStatus init(int concurreny); - protected: virtual OLAPStatus pick_rowsets_to_compact() = 0; virtual std::string compaction_name() const = 0; @@ -68,9 +66,6 @@ class Compaction { OLAPStatus check_version_continuity(const std::vector& rowsets); OLAPStatus check_correctness(const Merger::Statistics& stats); - // semaphore used to limit the concurrency of running compaction tasks - static Semaphore _concurrency_sem; - private: // get num rows from segment group meta of input rowsets. // return -1 if these are not alpha rowsets. diff --git a/be/src/olap/compaction_permit_limiter.cpp b/be/src/olap/compaction_permit_limiter.cpp index bbd1c823ffd358..3203296e8ae306 100644 --- a/be/src/olap/compaction_permit_limiter.cpp +++ b/be/src/olap/compaction_permit_limiter.cpp @@ -19,10 +19,6 @@ namespace doris { -uint32_t CompactionPermitLimiter::_total_permits; -uint32_t CompactionPermitLimiter::_used_permits; -bool CompactionPermitLimiter::_over_sold; - OLAPStatus CompactionPermitLimiter::init(uint32_t total_permits, bool over_sold) { _total_permits = total_permits; _over_sold = over_sold; @@ -33,7 +29,6 @@ OLAPStatus CompactionPermitLimiter::init(uint32_t total_permits, bool over_sold) bool CompactionPermitLimiter::request(uint32_t permits) { if (permits > _total_permits) { if (_over_sold) { - // wait_until used_permits==0 while(_used_permits != 0) { sleep(5); } diff --git a/be/src/olap/compaction_permit_limiter.h b/be/src/olap/compaction_permit_limiter.h index b5e637a37a2729..7f044e2e25787c 100644 --- a/be/src/olap/compaction_permit_limiter.h +++ b/be/src/olap/compaction_permit_limiter.h @@ -32,11 +32,11 @@ class CompactionPermitLimiter { CompactionPermitLimiter() {}; virtual ~CompactionPermitLimiter() { }; - static OLAPStatus init(uint32_t total_permits, bool _over_sold); + OLAPStatus init(uint32_t total_permits, bool _over_sold); - static bool request(uint32_t permits); + bool request(uint32_t permits); - static OLAPStatus release(uint32_t permits); + OLAPStatus release(uint32_t permits); inline uint32_t total_permits() const; inline void set_total_permits(uint32_t total_permits); @@ -47,9 +47,9 @@ class CompactionPermitLimiter { inline void set_over_sold(bool over_sold); private: - static uint32_t _total_permits; - static uint32_t _used_permits; - static bool _over_sold; + uint32_t _total_permits; + uint32_t _used_permits; + bool _over_sold; }; inline uint32_t CompactionPermitLimiter::total_permits() const { diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 2512fcd9fbf1e3..fa7a877a47a5eb 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -69,28 +69,17 @@ Status StorageEngine::start_bg_threads() { for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); } - int32_t data_dir_num = data_dirs.size(); // check cumulative compaction config _check_cumulative_compaction_config(); - // base and cumulative compaction threads - int32_t base_compaction_num_threads_per_disk = std::max(1, config::base_compaction_num_threads_per_disk); - int32_t cumulative_compaction_num_threads_per_disk = std::max(1, config::cumulative_compaction_num_threads_per_disk); - int32_t base_compaction_num_threads = base_compaction_num_threads_per_disk * data_dir_num; - int32_t cumulative_compaction_num_threads = cumulative_compaction_num_threads_per_disk * data_dir_num; - // calc the max concurrency of compaction tasks - int32_t max_compaction_concurrency = config::max_compaction_concurrency; - if (max_compaction_concurrency < 0 - || max_compaction_concurrency > base_compaction_num_threads + cumulative_compaction_num_threads + 1) { - // reserve 1 thread for manual execution - max_compaction_concurrency = base_compaction_num_threads + cumulative_compaction_num_threads + 1; - } - Compaction::init(max_compaction_concurrency); + int32_t max_thread_num = config::max_compaction_threads; + int32_t min_thread_num = config::min_compaction_threads; + ThreadPoolBuilder("CompactionTaskThreadPool").set_min_threads(min_thread_num).set_max_threads(max_thread_num).build(&_thread_pool_compaction); // compaction tasks producer thread uint32_t total_permits = config::total_permits_memory_for_compaction; - CompactionPermitLimiter::init(total_permits, true); + _permit_limiter.init(total_permits, true); RETURN_IF_ERROR( Thread::create("StorageEngine", "compaction_tasks_producer_thread", [this]() { this->_compaction_tasks_producer_callback(); }, @@ -163,7 +152,7 @@ void StorageEngine::_base_compaction_thread_callback(TabletSharedPtr tablet, uin LOG(INFO) << "try to start base compaction process!"; _perform_base_compaction(tablet); - CompactionPermitLimiter::release(permits); + _permit_limiter.release(permits); _map_disk_compaction_num[tablet->data_dir()] = _map_disk_compaction_num[tablet->data_dir()] - 1; } @@ -264,7 +253,7 @@ void StorageEngine::_cumulative_compaction_thread_callback(TabletSharedPtr table LOG(INFO) << "try to start cumulative compaction process!"; _perform_cumulative_compaction(tablet); - CompactionPermitLimiter::release(permits); + _permit_limiter.release(permits); _map_disk_compaction_num[tablet->data_dir()] = _map_disk_compaction_num[tablet->data_dir()] - 1; } @@ -352,10 +341,6 @@ Status StorageEngine::_compaction_tasks_producer_callback() { #endif LOG(INFO) << "try to start compaction producer process!"; - int thread_num = config::max_compaction_concurrency; - ThreadPoolBuilder("CompactionTaskThreadPool").set_min_threads(thread_num).set_max_threads(thread_num).build(&_thread_pool_compaction); - - // convert store map to vector std::vector data_dirs; for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); @@ -382,7 +367,7 @@ Status StorageEngine::_compaction_tasks_producer_callback() { } else { permits = tablets_compaction[i]->calc_base_compaction_score(); } - if (CompactionPermitLimiter::request(permits)) { + if (_permit_limiter.request(permits)) { if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_cumulative_compaction_thread_callback(tablets_compaction[i], permits);}); } else { diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index b5d189ebbdf414..81a7baeba65993 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -140,6 +140,7 @@ StorageEngine::~StorageEngine() { DEREGISTER_HOOK_METRIC(unused_rowsets_count); DEREGISTER_HOOK_METRIC(compaction_mem_current_consumption); _clear(); + _thread_pool_compaction->shutdown(); } void StorageEngine::load_data_dirs(const std::vector& data_dirs) { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 0d3f8f1b00a00e..06f726e3355963 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -37,13 +37,13 @@ #include "gen_cpp/BackendService_types.h" #include "gen_cpp/MasterService_types.h" #include "gutil/ref_counted.h" +#include "olap/compaction_permit_limiter.h" #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/tablet.h" #include "olap/olap_meta.h" #include "olap/options.h" #include "olap/tablet_manager.h" -#include "olap/compaction_permit_limiter.h" #include "olap/tablet_sync_service.h" #include "olap/txn_manager.h" #include "olap/task/engine_task.h" @@ -353,6 +353,8 @@ class StorageEngine { std::map _map_disk_compaction_num; std::unique_ptr _thread_pool_compaction; + + CompactionPermitLimiter _permit_limiter; }; } // namespace doris From 5f784bf9bef1f36cfdc3c531f2a5646461b6f219 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Mon, 21 Sep 2020 19:48:14 +0800 Subject: [PATCH 11/27] code refactor --- be/src/olap/olap_server.cpp | 9 ++++----- be/src/olap/storage_engine.h | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index fa7a877a47a5eb..0073e7139d0e1d 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -32,7 +32,6 @@ #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/storage_engine.h" -#include "olap/compaction_permit_limiter.h" #include "util/time.h" using std::string; @@ -145,7 +144,7 @@ void StorageEngine::_fd_cache_clean_callback() { } } -void StorageEngine::_base_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits) { +void StorageEngine::_base_compaction_task(TabletSharedPtr tablet, uint32_t permits) { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif @@ -246,7 +245,7 @@ void StorageEngine::_check_cumulative_compaction_config() { } } -void StorageEngine::_cumulative_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits) { +void StorageEngine::_cumulative_compaction_task(TabletSharedPtr tablet, uint32_t permits) { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif @@ -369,9 +368,9 @@ Status StorageEngine::_compaction_tasks_producer_callback() { } if (_permit_limiter.request(permits)) { if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_cumulative_compaction_thread_callback(tablets_compaction[i], permits);}); + _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_cumulative_compaction_task(tablets_compaction[i], permits);}); } else { - _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_base_compaction_thread_callback(tablets_compaction[i], permits);}); + _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_base_compaction_task(tablets_compaction[i], permits);}); } _map_disk_compaction_num[tablets_compaction[i]->data_dir()] = _map_disk_compaction_num[tablets_compaction[i]->data_dir()] + 1; } else { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 06f726e3355963..d11f4e8e8db447 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -215,11 +215,11 @@ class StorageEngine { void _unused_rowset_monitor_thread_callback(); // base compaction thread process function - void _base_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits); + void _base_compaction_task(TabletSharedPtr tablet, uint32_t permits); // check cumulative compaction config void _check_cumulative_compaction_config(); // cumulative process function - void _cumulative_compaction_thread_callback(TabletSharedPtr tablet, uint32_t permits); + void _cumulative_compaction_task(TabletSharedPtr tablet, uint32_t permits); // garbage sweep thread process function. clear snapshot and trash folder void _garbage_sweeper_thread_callback(); From e9406e7d1da7ac0b9448df54d4297878b5a7bd3b Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Tue, 22 Sep 2020 15:42:45 +0800 Subject: [PATCH 12/27] add metrics and comment for compaction limiter --- be/src/common/config.h | 2 +- be/src/olap/compaction_permit_limiter.h | 7 +++++- be/src/olap/data_dir.cpp | 22 +++++++++++++++++- be/src/olap/data_dir.h | 8 +++++++ be/src/olap/olap_server.cpp | 30 +++++++++++++++++++------ be/src/util/doris_metrics.h | 3 +++ 6 files changed, 62 insertions(+), 10 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index b3edbff5ee085d..2a8601e1b47d1c 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -314,7 +314,7 @@ namespace config { CONF_mInt32(max_compaction_threads, "10"); // This config can be set to limit memory for compaction. - CONF_mInt64(total_permits_memory_for_compaction, "10000") + CONF_mInt64(total_permits_for_compaction_score, "15000") // Threshold to logging compaction trace, in seconds. CONF_mInt32(base_compaction_trace_threshold, "10"); diff --git a/be/src/olap/compaction_permit_limiter.h b/be/src/olap/compaction_permit_limiter.h index 7f044e2e25787c..af31c2391d7bbe 100644 --- a/be/src/olap/compaction_permit_limiter.h +++ b/be/src/olap/compaction_permit_limiter.h @@ -26,7 +26,12 @@ namespace doris { -// This class is a base class for compaction management. +/* + This class is used to manage compaction permission. To some extent, it can be used to control the memory consumption. + "permits" should be applied before a compaction task can execute. When the sum of "permites" held by executing + compaction tasks reaches a set threshold, subsequent compaction task will be no longer allowed, until some "permits" + are released by some finished compaction tasks. "compaction score" for tablet is used as "permits" here. +*/ class CompactionPermitLimiter { public: CompactionPermitLimiter() {}; diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index 42afdd8232b610..c88cd3b60e4f13 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -59,6 +59,8 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(disks_total_capacity, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(disks_avail_capacity, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(disks_data_used_capacity, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(disks_state, MetricUnit::BYTES); +DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(disks_compaction_score, MetricUnit::NOUNIT); +DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(disks_compaction_num, MetricUnit::NOUNIT); static const char* const kMtabPath = "/etc/mtab"; static const char* const kTestFilePath = "/.testfile"; @@ -83,6 +85,8 @@ DataDir::DataDir(const std::string& path, int64_t capacity_bytes, INT_GAUGE_METRIC_REGISTER(_data_dir_metric_entity, disks_avail_capacity); INT_GAUGE_METRIC_REGISTER(_data_dir_metric_entity, disks_data_used_capacity); INT_GAUGE_METRIC_REGISTER(_data_dir_metric_entity, disks_state); + INT_GAUGE_METRIC_REGISTER(_data_dir_metric_entity, disks_compaction_score); + INT_GAUGE_METRIC_REGISTER(_data_dir_metric_entity, disks_compaction_num); } DataDir::~DataDir() { @@ -986,7 +990,7 @@ void DataDir::update_user_data_size(int64_t size) { bool DataDir::reach_capacity_limit(int64_t incoming_data_size) { double used_pct = (_disk_capacity_bytes - _available_bytes + incoming_data_size) / (double)_disk_capacity_bytes; - int64_t left_bytes = _disk_capacity_bytes - _available_bytes - incoming_data_size; + int64_t left_bytes = _available_bytes - incoming_data_size; if (used_pct >= config::storage_flood_stage_usage_percent / 100.0 && left_bytes <= config::storage_flood_stage_left_capacity_bytes) { @@ -996,4 +1000,20 @@ bool DataDir::reach_capacity_limit(int64_t incoming_data_size) { } return false; } + +void DataDir::update_disks_compaction_score(int64_t compaction_score) { + disks_compaction_score->set_value(compaction_score); +} + +int64_t DataDir::get_disks_compaction_score() { + return disks_compaction_score->value(); +} + +void DataDir::update_disks_compaction_num(int64_t compaction_num) { + disks_compaction_num->set_value(compaction_num); +} + +int64_t DataDir::get_disks_compaction_num() { + return disks_compaction_num->value(); +} } // namespace doris diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h index 0fc8b4a0969482..57bcdac39ec537 100644 --- a/be/src/olap/data_dir.h +++ b/be/src/olap/data_dir.h @@ -128,6 +128,12 @@ class DataDir { std::set tablet_set() { return _tablet_set; } + void update_disks_compaction_score(int64_t compaction_score); + int64_t get_disks_compaction_score(); + + void update_disks_compaction_num(int64_t compaction_num); + int64_t get_disks_compaction_num(); + private: std::string _cluster_id_path() const { return _path + CLUSTER_ID_PREFIX; } Status _init_cluster_id(); @@ -201,6 +207,8 @@ class DataDir { IntGauge* disks_avail_capacity; IntGauge* disks_data_used_capacity; IntGauge* disks_state; + IntGauge* disks_compaction_score; + IntGauge* disks_compaction_num; }; } // namespace doris diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 0073e7139d0e1d..1f3cf5b4f1e326 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -32,6 +32,7 @@ #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/storage_engine.h" +#include "util/doris_metrics.h" #include "util/time.h" using std::string; @@ -67,7 +68,11 @@ Status StorageEngine::start_bg_threads() { std::vector data_dirs; for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); + tmp_store.second->update_disks_compaction_score(0); + tmp_store.second->update_disks_compaction_num(0); } + DorisMetrics::instance()->total_compaction_score->set_value(0); + DorisMetrics::instance()->total_compaction_num->set_value(0); // check cumulative compaction config _check_cumulative_compaction_config(); @@ -77,7 +82,7 @@ Status StorageEngine::start_bg_threads() { ThreadPoolBuilder("CompactionTaskThreadPool").set_min_threads(min_thread_num).set_max_threads(max_thread_num).build(&_thread_pool_compaction); // compaction tasks producer thread - uint32_t total_permits = config::total_permits_memory_for_compaction; + uint32_t total_permits = config::total_permits_for_compaction_score; _permit_limiter.init(total_permits, true); RETURN_IF_ERROR( Thread::create("StorageEngine", "compaction_tasks_producer_thread", @@ -150,9 +155,16 @@ void StorageEngine::_base_compaction_task(TabletSharedPtr tablet, uint32_t permi #endif LOG(INFO) << "try to start base compaction process!"; + tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() + permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + 1); + DorisMetrics::instance()->total_compaction_score->increment(permits); + DorisMetrics::instance()->total_compaction_num->increment(1); _perform_base_compaction(tablet); _permit_limiter.release(permits); - _map_disk_compaction_num[tablet->data_dir()] = _map_disk_compaction_num[tablet->data_dir()] - 1; + tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() - permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - 1); + DorisMetrics::instance()->total_compaction_score->increment(-permits); + DorisMetrics::instance()->total_compaction_num->increment(-1); } void StorageEngine::_garbage_sweeper_thread_callback() { @@ -251,9 +263,16 @@ void StorageEngine::_cumulative_compaction_task(TabletSharedPtr tablet, uint32_t #endif LOG(INFO) << "try to start cumulative compaction process!"; + tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() + permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + 1); + DorisMetrics::instance()->total_compaction_score->increment(permits); + DorisMetrics::instance()->total_compaction_num->increment(1); _perform_cumulative_compaction(tablet); _permit_limiter.release(permits); - _map_disk_compaction_num[tablet->data_dir()] = _map_disk_compaction_num[tablet->data_dir()] - 1; + tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() - permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - 1); + DorisMetrics::instance()->total_compaction_score->increment(-permits); + DorisMetrics::instance()->total_compaction_num->increment(-1); } void StorageEngine::_unused_rowset_monitor_thread_callback() { @@ -343,7 +362,6 @@ Status StorageEngine::_compaction_tasks_producer_callback() { std::vector data_dirs; for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); - _map_disk_compaction_num[tmp_store.second] = 0; } int round = 0; @@ -357,9 +375,8 @@ Status StorageEngine::_compaction_tasks_producer_callback() { round = 0; } vector tablets_compaction = _compaction_tasks_generator(compaction_type, data_dirs); - for (int i = 0; i < tablets_compaction.size(); i++) { - if(_map_disk_compaction_num[tablets_compaction[i]->data_dir()] == 0) { + if(tablets_compaction[i]->data_dir()->get_disks_compaction_num() == 0) { uint32_t permits; if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { permits = tablets_compaction[i]->calc_cumulative_compaction_score(); @@ -372,7 +389,6 @@ Status StorageEngine::_compaction_tasks_producer_callback() { } else { _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_base_compaction_task(tablets_compaction[i], permits);}); } - _map_disk_compaction_num[tablets_compaction[i]->data_dir()] = _map_disk_compaction_num[tablets_compaction[i]->data_dir()] + 1; } else { continue; } diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h index 3ffbe746d88203..5d357908786e78 100644 --- a/be/src/util/doris_metrics.h +++ b/be/src/util/doris_metrics.h @@ -133,6 +133,9 @@ class DorisMetrics { IntGauge* tablet_cumulative_max_compaction_score; IntGauge* tablet_base_max_compaction_score; + IntGauge* total_compaction_score; + IntGauge* total_compaction_num; + // The following metrics will be calculated // by metric calculator IntGauge* push_request_write_bytes_per_second; From f7b614c3f7932d90edd6c5457533c544fe0c1c29 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Tue, 22 Sep 2020 18:38:22 +0800 Subject: [PATCH 13/27] modify metrics --- be/src/olap/olap_server.cpp | 3 ++- be/src/util/doris_metrics.cpp | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 1f3cf5b4f1e326..2ca0e1de946d22 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -374,9 +374,10 @@ Status StorageEngine::_compaction_tasks_producer_callback() { compaction_type = CompactionType::BASE_COMPACTION; round = 0; } + LOG(INFO) << "try to generate a batch of compaction tasks!"; vector tablets_compaction = _compaction_tasks_generator(compaction_type, data_dirs); for (int i = 0; i < tablets_compaction.size(); i++) { - if(tablets_compaction[i]->data_dir()->get_disks_compaction_num() == 0) { + if(tablets_compaction[i]->data_dir()->get_disks_compaction_num() < 5) { uint32_t permits; if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { permits = tablets_compaction[i]->calc_cumulative_compaction_score(); diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp index 023074a66e9d0f..beae32a27e7d7f 100644 --- a/be/src/util/doris_metrics.cpp +++ b/be/src/util/doris_metrics.cpp @@ -110,6 +110,9 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(process_fd_num_limit_hard, MetricUnit::NOUNIT DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(tablet_cumulative_max_compaction_score, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(tablet_base_max_compaction_score, MetricUnit::NOUNIT); +DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(total_compaction_score, MetricUnit::NOUNIT); +DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(total_compaction_num, MetricUnit::NOUNIT); + DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(push_request_write_bytes_per_second, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(query_scan_bytes_per_second, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(max_disk_io_util_percent, MetricUnit::PERCENT); @@ -210,6 +213,9 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_cumulative_max_compaction_score); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_base_max_compaction_score); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_score); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_num); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, push_request_write_bytes_per_second); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, query_scan_bytes_per_second); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, max_disk_io_util_percent); From fbe9b575485686d351e394c85760df0df4c9541e Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Tue, 22 Sep 2020 21:26:16 +0800 Subject: [PATCH 14/27] modify metrics --- be/src/olap/olap_server.cpp | 54 +++++++++++++++++++++--------------- be/src/olap/storage_engine.h | 1 + 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 2ca0e1de946d22..117cff7eee940b 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -68,11 +68,11 @@ Status StorageEngine::start_bg_threads() { std::vector data_dirs; for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); - tmp_store.second->update_disks_compaction_score(0); - tmp_store.second->update_disks_compaction_num(0); +// tmp_store.second->update_disks_compaction_score(0); +// tmp_store.second->update_disks_compaction_num(0); } - DorisMetrics::instance()->total_compaction_score->set_value(0); - DorisMetrics::instance()->total_compaction_num->set_value(0); +// DorisMetrics::instance()->total_compaction_score->set_value(0); +// DorisMetrics::instance()->total_compaction_num->set_value(0); // check cumulative compaction config _check_cumulative_compaction_config(); @@ -154,17 +154,22 @@ void StorageEngine::_base_compaction_task(TabletSharedPtr tablet, uint32_t permi ProfilerRegisterThread(); #endif LOG(INFO) << "try to start base compaction process!"; - - tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() + permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + 1); - DorisMetrics::instance()->total_compaction_score->increment(permits); - DorisMetrics::instance()->total_compaction_num->increment(1); + { + std::unique_lock lock(_mutex_compaction_metrics); + tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() + permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + 1); + DorisMetrics::instance()->total_compaction_score->increment(permits); + DorisMetrics::instance()->total_compaction_num->increment(1); + } _perform_base_compaction(tablet); _permit_limiter.release(permits); - tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() - permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - 1); - DorisMetrics::instance()->total_compaction_score->increment(-permits); - DorisMetrics::instance()->total_compaction_num->increment(-1); + { + std::unique_lock lock(_mutex_compaction_metrics); + tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() - permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - 1); + DorisMetrics::instance()->total_compaction_score->increment(-permits); + DorisMetrics::instance()->total_compaction_num->increment(-1); + } } void StorageEngine::_garbage_sweeper_thread_callback() { @@ -262,17 +267,22 @@ void StorageEngine::_cumulative_compaction_task(TabletSharedPtr tablet, uint32_t ProfilerRegisterThread(); #endif LOG(INFO) << "try to start cumulative compaction process!"; - - tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() + permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + 1); - DorisMetrics::instance()->total_compaction_score->increment(permits); - DorisMetrics::instance()->total_compaction_num->increment(1); + { + std::unique_lock lock(_mutex_compaction_metrics); + tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() + permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + 1); + DorisMetrics::instance()->total_compaction_score->increment(permits); + DorisMetrics::instance()->total_compaction_num->increment(1); + } _perform_cumulative_compaction(tablet); _permit_limiter.release(permits); - tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() - permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - 1); - DorisMetrics::instance()->total_compaction_score->increment(-permits); - DorisMetrics::instance()->total_compaction_num->increment(-1); + { + std::unique_lock lock(_mutex_compaction_metrics); + tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() - permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - 1); + DorisMetrics::instance()->total_compaction_score->increment(-permits); + DorisMetrics::instance()->total_compaction_num->increment(-1); + } } void StorageEngine::_unused_rowset_monitor_thread_callback() { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index d11f4e8e8db447..c1fbdd7446a232 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -355,6 +355,7 @@ class StorageEngine { std::unique_ptr _thread_pool_compaction; CompactionPermitLimiter _permit_limiter; + std::mutex _mutex_compaction_metrics; }; } // namespace doris From 7200398e5ea4c427b42afc9afde90481e62f173f Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Wed, 23 Sep 2020 09:59:51 +0800 Subject: [PATCH 15/27] modify metrics --- be/src/util/doris_metrics.cpp | 4 ++-- be/src/util/doris_metrics.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp index beae32a27e7d7f..a8c29f34108923 100644 --- a/be/src/util/doris_metrics.cpp +++ b/be/src/util/doris_metrics.cpp @@ -213,8 +213,8 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_cumulative_max_compaction_score); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_base_max_compaction_score); - INT_GAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_score); - INT_GAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_num); + INT_UGAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_score); + INT_UGAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_num); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, push_request_write_bytes_per_second); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, query_scan_bytes_per_second); diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h index 5d357908786e78..9398c3e0d5d1f5 100644 --- a/be/src/util/doris_metrics.h +++ b/be/src/util/doris_metrics.h @@ -133,8 +133,8 @@ class DorisMetrics { IntGauge* tablet_cumulative_max_compaction_score; IntGauge* tablet_base_max_compaction_score; - IntGauge* total_compaction_score; - IntGauge* total_compaction_num; + UIntGauge* total_compaction_score; + UIntGauge* total_compaction_num; // The following metrics will be calculated // by metric calculator From 8652b55ee3148a83c2f27ed78941f2a20b455743 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Wed, 23 Sep 2020 11:15:47 +0800 Subject: [PATCH 16/27] modify metrics --- be/src/olap/olap_server.cpp | 16 ++++++++++------ be/src/util/doris_metrics.cpp | 4 ++-- be/src/util/doris_metrics.h | 4 ++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 117cff7eee940b..568685c48ec989 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -158,16 +158,19 @@ void StorageEngine::_base_compaction_task(TabletSharedPtr tablet, uint32_t permi std::unique_lock lock(_mutex_compaction_metrics); tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() + permits); tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + 1); - DorisMetrics::instance()->total_compaction_score->increment(permits); + //DorisMetrics::instance()->total_compaction_score->increment(permits); + DorisMetrics::instance()->total_compaction_score->set_value(DorisMetrics::instance()->total_compaction_score->value() + permits); DorisMetrics::instance()->total_compaction_num->increment(1); } _perform_base_compaction(tablet); + sleep(25); _permit_limiter.release(permits); { std::unique_lock lock(_mutex_compaction_metrics); tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() - permits); tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - 1); - DorisMetrics::instance()->total_compaction_score->increment(-permits); + //DorisMetrics::instance()->total_compaction_score->increment(-permits); + DorisMetrics::instance()->total_compaction_score->set_value(DorisMetrics::instance()->total_compaction_score->value() + permits); DorisMetrics::instance()->total_compaction_num->increment(-1); } } @@ -271,16 +274,19 @@ void StorageEngine::_cumulative_compaction_task(TabletSharedPtr tablet, uint32_t std::unique_lock lock(_mutex_compaction_metrics); tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() + permits); tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + 1); - DorisMetrics::instance()->total_compaction_score->increment(permits); + //DorisMetrics::instance()->total_compaction_score->increment(permits); + DorisMetrics::instance()->total_compaction_score->set_value(DorisMetrics::instance()->total_compaction_score->value() + permits); DorisMetrics::instance()->total_compaction_num->increment(1); } _perform_cumulative_compaction(tablet); + sleep(25); _permit_limiter.release(permits); { std::unique_lock lock(_mutex_compaction_metrics); tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() - permits); tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - 1); - DorisMetrics::instance()->total_compaction_score->increment(-permits); + //DorisMetrics::instance()->total_compaction_score->increment(-permits); + DorisMetrics::instance()->total_compaction_score->set_value(DorisMetrics::instance()->total_compaction_score->value() - permits); DorisMetrics::instance()->total_compaction_num->increment(-1); } } @@ -400,8 +406,6 @@ Status StorageEngine::_compaction_tasks_producer_callback() { } else { _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_base_compaction_task(tablets_compaction[i], permits);}); } - } else { - continue; } } } diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp index a8c29f34108923..beae32a27e7d7f 100644 --- a/be/src/util/doris_metrics.cpp +++ b/be/src/util/doris_metrics.cpp @@ -213,8 +213,8 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_cumulative_max_compaction_score); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_base_max_compaction_score); - INT_UGAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_score); - INT_UGAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_num); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_score); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_num); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, push_request_write_bytes_per_second); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, query_scan_bytes_per_second); diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h index 9398c3e0d5d1f5..5d357908786e78 100644 --- a/be/src/util/doris_metrics.h +++ b/be/src/util/doris_metrics.h @@ -133,8 +133,8 @@ class DorisMetrics { IntGauge* tablet_cumulative_max_compaction_score; IntGauge* tablet_base_max_compaction_score; - UIntGauge* total_compaction_score; - UIntGauge* total_compaction_num; + IntGauge* total_compaction_score; + IntGauge* total_compaction_num; // The following metrics will be calculated // by metric calculator From 644b12606dbe27dacceb2f5b2aae35ce0aeb2449 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Wed, 23 Sep 2020 15:06:00 +0800 Subject: [PATCH 17/27] modify config --- be/src/common/config.h | 12 ++++++------ be/src/olap/compaction_permit_limiter.cpp | 6 ++---- be/src/olap/compaction_permit_limiter.h | 14 +++++--------- be/src/olap/olap_server.cpp | 6 ++++-- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index 2a8601e1b47d1c..cad523b81ae766 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -261,13 +261,8 @@ namespace config { // whether to disable page cache feature in storage CONF_Bool(disable_storage_page_cache, "false"); - // be policy - // whether disable automatic compaction task - CONF_mBool(disable_auto_compaction, "false"); - // CONF_Int64(base_compaction_start_hour, "20"); // CONF_Int64(base_compaction_end_hour, "7"); - CONF_mInt32(base_compaction_check_interval_seconds, "60"); CONF_mInt64(base_compaction_num_cumulative_deltas, "5"); CONF_mDouble(base_cumulative_delta_ratio, "0.3"); CONF_mInt64(base_compaction_interval_seconds_since_last_operation, "86400"); @@ -295,7 +290,6 @@ namespace config { CONF_mInt64(cumulative_size_based_compaction_lower_size_mbytes, "64"); // cumulative compaction policy: max delta file's size unit:B - CONF_mInt32(cumulative_compaction_check_interval_seconds, "10"); CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5"); CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000"); CONF_mInt64(cumulative_compaction_budgeted_bytes, "104857600"); @@ -316,6 +310,12 @@ namespace config { // This config can be set to limit memory for compaction. CONF_mInt64(total_permits_for_compaction_score, "15000") + // sleep time of compaction tasks producer thread after each tasks generation, in seconds. + CONF_mInt32(generate_compaction_tasks_interval_seconds, "5") + + // compaction task number for per disk + CONF_mInt32(compaction_task_num_per_disk, "5"); + // Threshold to logging compaction trace, in seconds. CONF_mInt32(base_compaction_trace_threshold, "10"); CONF_mInt32(cumulative_compaction_trace_threshold, "2"); diff --git a/be/src/olap/compaction_permit_limiter.cpp b/be/src/olap/compaction_permit_limiter.cpp index 3203296e8ae306..4bc49108a3b27a 100644 --- a/be/src/olap/compaction_permit_limiter.cpp +++ b/be/src/olap/compaction_permit_limiter.cpp @@ -19,11 +19,10 @@ namespace doris { -OLAPStatus CompactionPermitLimiter::init(uint32_t total_permits, bool over_sold) { +void CompactionPermitLimiter::init(uint32_t total_permits, bool over_sold) { _total_permits = total_permits; _over_sold = over_sold; _used_permits = 0; - return OLAP_SUCCESS; } bool CompactionPermitLimiter::request(uint32_t permits) { @@ -46,12 +45,11 @@ bool CompactionPermitLimiter::request(uint32_t permits) { } } -OLAPStatus CompactionPermitLimiter::release(uint32_t permits) { +void CompactionPermitLimiter::release(uint32_t permits) { if (permits > _total_permits) { _used_permits = 0; } else { _used_permits = _used_permits - permits; } - return OLAP_SUCCESS; } } // namespace doris diff --git a/be/src/olap/compaction_permit_limiter.h b/be/src/olap/compaction_permit_limiter.h index af31c2391d7bbe..0dbae649db625a 100644 --- a/be/src/olap/compaction_permit_limiter.h +++ b/be/src/olap/compaction_permit_limiter.h @@ -15,10 +15,7 @@ // specific language governing permissions and limitations // under the License. -#ifndef DORIS_BE_SRC_OLAP_COMPACTION_PERMIT_LIMITER_H -#define DORIS_BE_SRC_OLAP_COMPACTION_PERMIT_LIMITER_H - -#include +#pragma once #include "olap/olap_common.h" #include "olap/olap_define.h" @@ -34,14 +31,14 @@ namespace doris { */ class CompactionPermitLimiter { public: - CompactionPermitLimiter() {}; - virtual ~CompactionPermitLimiter() { }; + CompactionPermitLimiter() {} + virtual ~CompactionPermitLimiter() {} - OLAPStatus init(uint32_t total_permits, bool _over_sold); + void init(uint32_t total_permits, bool _over_sold); bool request(uint32_t permits); - OLAPStatus release(uint32_t permits); + void release(uint32_t permits); inline uint32_t total_permits() const; inline void set_total_permits(uint32_t total_permits); @@ -78,4 +75,3 @@ inline void CompactionPermitLimiter::set_over_sold(bool over_sold) { } } // namespace doris -#endif // DORIS_BE_SRC_OLAP_COMPACTION_PERMIT_LIMITER_H diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 568685c48ec989..f2b1b56a13d219 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -375,6 +375,8 @@ Status StorageEngine::_compaction_tasks_producer_callback() { #endif LOG(INFO) << "try to start compaction producer process!"; + //int32_t interval_tasks_generator = config::generate_compaction_tasks_interval_seconds; + //int32_t disk_task_num = config::compaction_task_num_per_disk; std::vector data_dirs; for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); @@ -393,7 +395,7 @@ Status StorageEngine::_compaction_tasks_producer_callback() { LOG(INFO) << "try to generate a batch of compaction tasks!"; vector tablets_compaction = _compaction_tasks_generator(compaction_type, data_dirs); for (int i = 0; i < tablets_compaction.size(); i++) { - if(tablets_compaction[i]->data_dir()->get_disks_compaction_num() < 5) { + if(tablets_compaction[i]->data_dir()->get_disks_compaction_num() < config::compaction_task_num_per_disk) { uint32_t permits; if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { permits = tablets_compaction[i]->calc_cumulative_compaction_score(); @@ -409,7 +411,7 @@ Status StorageEngine::_compaction_tasks_producer_callback() { } } } - sleep(5); + sleep(config::generate_compaction_tasks_interval_seconds); } while (true); } From 6ce8595e0b183ed9a1ebaa8b52c66db2dd42ffcc Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Wed, 23 Sep 2020 20:08:57 +0800 Subject: [PATCH 18/27] modify config --- be/src/common/config.h | 12 ++- be/src/olap/compaction_permit_limiter.cpp | 30 +++--- be/src/olap/compaction_permit_limiter.h | 52 +++------- be/src/olap/olap_server.cpp | 119 ++++++++++------------ be/src/olap/storage_engine.cpp | 2 +- be/src/olap/storage_engine.h | 25 ++--- be/src/olap/tablet.cpp | 10 ++ be/src/olap/tablet.h | 1 + 8 files changed, 107 insertions(+), 144 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index cad523b81ae766..55d80a74ffe26f 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -307,15 +307,21 @@ namespace config { CONF_mInt32(min_compaction_threads, "10"); CONF_mInt32(max_compaction_threads, "10"); - // This config can be set to limit memory for compaction. + // The upper limit of "permites" held by all compaction tasks. This config can be set to limit memory consumption for compaction. CONF_mInt64(total_permits_for_compaction_score, "15000") - // sleep time of compaction tasks producer thread after each tasks generation, in seconds. + // Whether compaction task is allowed to start when compaction score of current tablet is out of upper limit. + CONF_mBool(enable_over_sold, "true"); + + // Sleep time of compaction tasks producer thread after each tasks generation, in seconds. CONF_mInt32(generate_compaction_tasks_interval_seconds, "5") - // compaction task number for per disk + // Compaction task number for per disk. CONF_mInt32(compaction_task_num_per_disk, "5"); + // How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation. + CONF_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round, "9"); + // Threshold to logging compaction trace, in seconds. CONF_mInt32(base_compaction_trace_threshold, "10"); CONF_mInt32(cumulative_compaction_trace_threshold, "2"); diff --git a/be/src/olap/compaction_permit_limiter.cpp b/be/src/olap/compaction_permit_limiter.cpp index 4bc49108a3b27a..3fd40a2f3033c7 100644 --- a/be/src/olap/compaction_permit_limiter.cpp +++ b/be/src/olap/compaction_permit_limiter.cpp @@ -19,25 +19,23 @@ namespace doris { -void CompactionPermitLimiter::init(uint32_t total_permits, bool over_sold) { - _total_permits = total_permits; - _over_sold = over_sold; - _used_permits = 0; -} +CompactionPermitLimiter::CompactionPermitLimiter() : _used_permits(0) {} bool CompactionPermitLimiter::request(uint32_t permits) { - if (permits > _total_permits) { - if (_over_sold) { - while(_used_permits != 0) { - sleep(5); - } - _used_permits = _total_permits; + if (permits > config::total_permits_for_compaction_score) { + if (config::enable_over_sold) { + std::unique_lock lock(_over_sold_mutex); + _cv.wait(lock, [=] { + return _used_permits == 0 || + _used_permits + permits < config::total_permits_for_compaction_score; + }); + _used_permits += permits; return true; } else { return false; } } else { - if ((_total_permits - _used_permits) < permits) { + if (_used_permits + permits > config::total_permits_for_compaction_score) { return false; } _used_permits += permits; @@ -46,10 +44,6 @@ bool CompactionPermitLimiter::request(uint32_t permits) { } void CompactionPermitLimiter::release(uint32_t permits) { - if (permits > _total_permits) { - _used_permits = 0; - } else { - _used_permits = _used_permits - permits; - } + _used_permits -= permits; } -} // namespace doris +} // namespace doris diff --git a/be/src/olap/compaction_permit_limiter.h b/be/src/olap/compaction_permit_limiter.h index 0dbae649db625a..8e225a8bf04e71 100644 --- a/be/src/olap/compaction_permit_limiter.h +++ b/be/src/olap/compaction_permit_limiter.h @@ -17,61 +17,33 @@ #pragma once -#include "olap/olap_common.h" -#include "olap/olap_define.h" +#include +#include + +#include "common/config.h" #include "olap/utils.h" namespace doris { /* - This class is used to manage compaction permission. To some extent, it can be used to control the memory consumption. + This class is used to control compaction permission. To some extent, it can be used to control memory consumption. "permits" should be applied before a compaction task can execute. When the sum of "permites" held by executing - compaction tasks reaches a set threshold, subsequent compaction task will be no longer allowed, until some "permits" + compaction tasks reaches a threshold, subsequent compaction task will be no longer allowed, until some "permits" are released by some finished compaction tasks. "compaction score" for tablet is used as "permits" here. */ class CompactionPermitLimiter { public: - CompactionPermitLimiter() {} + CompactionPermitLimiter(); virtual ~CompactionPermitLimiter() {} - void init(uint32_t total_permits, bool _over_sold); - bool request(uint32_t permits); void release(uint32_t permits); - inline uint32_t total_permits() const; - inline void set_total_permits(uint32_t total_permits); - - inline uint32_t used_permits() const; - - inline bool is_over_sold() const; - inline void set_over_sold(bool over_sold); - private: - uint32_t _total_permits; - uint32_t _used_permits; - bool _over_sold; + // sum of "permites" held by executing compaction tasks currently + AtomicInt64 _used_permits; + std::mutex _over_sold_mutex; + std::condition_variable _cv; }; - -inline uint32_t CompactionPermitLimiter::total_permits() const { - return _total_permits; -} - -inline void CompactionPermitLimiter::set_total_permits(uint32_t total_permits) { - _total_permits = total_permits; -} - -inline uint32_t CompactionPermitLimiter::used_permits() const { - return _used_permits; -} - -inline bool CompactionPermitLimiter::is_over_sold() const { - return _over_sold; -} - -inline void CompactionPermitLimiter::set_over_sold(bool over_sold) { - _over_sold = over_sold; -} -} // namespace doris - +} // namespace doris diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index f2b1b56a13d219..87f11ceadbd857 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -63,27 +63,21 @@ Status StorageEngine::start_bg_threads() { &_disk_stat_monitor_thread)); LOG(INFO) << "disk stat monitor thread started"; - + // convert store map to vector std::vector data_dirs; for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); -// tmp_store.second->update_disks_compaction_score(0); -// tmp_store.second->update_disks_compaction_num(0); } -// DorisMetrics::instance()->total_compaction_score->set_value(0); -// DorisMetrics::instance()->total_compaction_num->set_value(0); // check cumulative compaction config _check_cumulative_compaction_config(); int32_t max_thread_num = config::max_compaction_threads; int32_t min_thread_num = config::min_compaction_threads; - ThreadPoolBuilder("CompactionTaskThreadPool").set_min_threads(min_thread_num).set_max_threads(max_thread_num).build(&_thread_pool_compaction); + ThreadPoolBuilder("CompactionTaskThreadPool").set_min_threads(min_thread_num).set_max_threads(max_thread_num).build(&_compaction_thread_pool); // compaction tasks producer thread - uint32_t total_permits = config::total_permits_for_compaction_score; - _permit_limiter.init(total_permits, true); RETURN_IF_ERROR( Thread::create("StorageEngine", "compaction_tasks_producer_thread", [this]() { this->_compaction_tasks_producer_callback(); }, @@ -150,29 +144,23 @@ void StorageEngine::_fd_cache_clean_callback() { } void StorageEngine::_base_compaction_task(TabletSharedPtr tablet, uint32_t permits) { -#ifdef GOOGLE_PROFILER - ProfilerRegisterThread(); -#endif - LOG(INFO) << "try to start base compaction process!"; - { - std::unique_lock lock(_mutex_compaction_metrics); - tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() + permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + 1); - //DorisMetrics::instance()->total_compaction_score->increment(permits); - DorisMetrics::instance()->total_compaction_score->set_value(DorisMetrics::instance()->total_compaction_score->value() + permits); - DorisMetrics::instance()->total_compaction_num->increment(1); - } + tablet->data_dir()->update_disks_compaction_score( + tablet->data_dir()->get_disks_compaction_score() + permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + + 1); + DorisMetrics::instance()->total_compaction_score->set_value( + DorisMetrics::instance()->total_compaction_score->value() + permits); + DorisMetrics::instance()->total_compaction_num->increment(1); _perform_base_compaction(tablet); sleep(25); _permit_limiter.release(permits); - { - std::unique_lock lock(_mutex_compaction_metrics); - tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() - permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - 1); - //DorisMetrics::instance()->total_compaction_score->increment(-permits); - DorisMetrics::instance()->total_compaction_score->set_value(DorisMetrics::instance()->total_compaction_score->value() + permits); - DorisMetrics::instance()->total_compaction_num->increment(-1); - } + tablet->data_dir()->update_disks_compaction_score( + tablet->data_dir()->get_disks_compaction_score() - permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - + 1); + DorisMetrics::instance()->total_compaction_score->set_value( + DorisMetrics::instance()->total_compaction_score->value() + permits); + DorisMetrics::instance()->total_compaction_num->increment(-1); } void StorageEngine::_garbage_sweeper_thread_callback() { @@ -266,29 +254,23 @@ void StorageEngine::_check_cumulative_compaction_config() { } void StorageEngine::_cumulative_compaction_task(TabletSharedPtr tablet, uint32_t permits) { -#ifdef GOOGLE_PROFILER - ProfilerRegisterThread(); -#endif - LOG(INFO) << "try to start cumulative compaction process!"; - { - std::unique_lock lock(_mutex_compaction_metrics); - tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() + permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + 1); - //DorisMetrics::instance()->total_compaction_score->increment(permits); - DorisMetrics::instance()->total_compaction_score->set_value(DorisMetrics::instance()->total_compaction_score->value() + permits); - DorisMetrics::instance()->total_compaction_num->increment(1); - } + tablet->data_dir()->update_disks_compaction_score( + tablet->data_dir()->get_disks_compaction_score() + permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + + 1); + DorisMetrics::instance()->total_compaction_score->set_value( + DorisMetrics::instance()->total_compaction_score->value() + permits); + DorisMetrics::instance()->total_compaction_num->increment(1); _perform_cumulative_compaction(tablet); sleep(25); _permit_limiter.release(permits); - { - std::unique_lock lock(_mutex_compaction_metrics); - tablet->data_dir()->update_disks_compaction_score(tablet->data_dir()->get_disks_compaction_score() - permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - 1); - //DorisMetrics::instance()->total_compaction_score->increment(-permits); - DorisMetrics::instance()->total_compaction_score->set_value(DorisMetrics::instance()->total_compaction_score->value() - permits); - DorisMetrics::instance()->total_compaction_num->increment(-1); - } + tablet->data_dir()->update_disks_compaction_score( + tablet->data_dir()->get_disks_compaction_score() - permits); + tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - + 1); + DorisMetrics::instance()->total_compaction_score->set_value( + DorisMetrics::instance()->total_compaction_score->value() - permits); + DorisMetrics::instance()->total_compaction_num->increment(-1); } void StorageEngine::_unused_rowset_monitor_thread_callback() { @@ -318,7 +300,7 @@ void StorageEngine::_path_gc_thread_callback(DataDir* data_dir) { do { LOG(INFO) << "try to perform path gc by tablet!"; data_dir->perform_path_gc_by_tablet(); - + LOG(INFO) << "try to perform path gc by rowsetid!"; data_dir->perform_path_gc_by_rowsetid(); @@ -369,14 +351,12 @@ void StorageEngine::_tablet_checkpoint_callback(DataDir* data_dir) { } while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(interval))); } -Status StorageEngine::_compaction_tasks_producer_callback() { +void StorageEngine::_compaction_tasks_producer_callback() { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif LOG(INFO) << "try to start compaction producer process!"; - //int32_t interval_tasks_generator = config::generate_compaction_tasks_interval_seconds; - //int32_t disk_task_num = config::compaction_task_num_per_disk; std::vector data_dirs; for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); @@ -385,7 +365,7 @@ Status StorageEngine::_compaction_tasks_producer_callback() { int round = 0; CompactionType compaction_type; do { - if(round < 9) { + if (round < config::cumulative_compaction_rounds_for_each_base_compaction_round) { compaction_type = CompactionType::CUMULATIVE_COMPACTION; round++; } else { @@ -393,34 +373,37 @@ Status StorageEngine::_compaction_tasks_producer_callback() { round = 0; } LOG(INFO) << "try to generate a batch of compaction tasks!"; - vector tablets_compaction = _compaction_tasks_generator(compaction_type, data_dirs); - for (int i = 0; i < tablets_compaction.size(); i++) { - if(tablets_compaction[i]->data_dir()->get_disks_compaction_num() < config::compaction_task_num_per_disk) { - uint32_t permits; - if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - permits = tablets_compaction[i]->calc_cumulative_compaction_score(); - } else { - permits = tablets_compaction[i]->calc_base_compaction_score(); - } + vector tablets_compaction = + _compaction_tasks_generator(compaction_type, data_dirs); + for (const auto& tablet : tablets_compaction) { + if (tablet->data_dir()->get_disks_compaction_num() < + config::compaction_task_num_per_disk) { + uint32_t permits = tablet->calc_compaction_score(compaction_type); if (_permit_limiter.request(permits)) { if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_cumulative_compaction_task(tablets_compaction[i], permits);}); + _compaction_thread_pool->submit_func([this, tablet, permits]() { + this->_cumulative_compaction_task(tablet, permits); + }); } else { - _thread_pool_compaction->submit_func([this, i, tablets_compaction, permits]() {this->_base_compaction_task(tablets_compaction[i], permits);}); + _compaction_thread_pool->submit_func([this, tablet, permits]() { + this->_base_compaction_task(tablet, permits); + }); } } } } - sleep(config::generate_compaction_tasks_interval_seconds); - } while (true); + } while (!_stop_background_threads_latch.wait_for( + MonoDelta::FromSeconds(config::generate_compaction_tasks_interval_seconds))); } -vector StorageEngine::_compaction_tasks_generator(CompactionType compaction_type, std::vector data_dirs) { +vector StorageEngine::_compaction_tasks_generator( + CompactionType compaction_type, std::vector data_dirs) { vector tablets_compaction; std::random_shuffle(data_dirs.begin(), data_dirs.end()); for (auto data_dir : data_dirs) { if (!data_dir->reach_capacity_limit(0)) { - TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction(compaction_type, data_dir); + TabletSharedPtr tablet = + _tablet_manager->find_best_tablet_to_compaction(compaction_type, data_dir); tablets_compaction.emplace_back(tablet); } } diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 81a7baeba65993..020b40fca3ec8f 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -140,7 +140,7 @@ StorageEngine::~StorageEngine() { DEREGISTER_HOOK_METRIC(unused_rowsets_count); DEREGISTER_HOOK_METRIC(compaction_mem_current_consumption); _clear(); - _thread_pool_compaction->shutdown(); + _compaction_thread_pool->shutdown(); } void StorageEngine::load_data_dirs(const std::vector& data_dirs) { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index c1fbdd7446a232..67acd1610272a2 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -18,18 +18,18 @@ #ifndef DORIS_BE_SRC_OLAP_STORAGE_ENGINE_H #define DORIS_BE_SRC_OLAP_STORAGE_ENGINE_H +#include #include #include #include #include -#include #include #include -#include #include +#include -#include #include +#include #include "agent/status.h" #include "common/status.h" @@ -38,23 +38,22 @@ #include "gen_cpp/MasterService_types.h" #include "gutil/ref_counted.h" #include "olap/compaction_permit_limiter.h" +#include "olap/fs/fs_util.h" #include "olap/olap_common.h" #include "olap/olap_define.h" -#include "olap/tablet.h" #include "olap/olap_meta.h" #include "olap/options.h" +#include "olap/rowset/rowset_id_generator.h" +#include "olap/tablet.h" #include "olap/tablet_manager.h" #include "olap/tablet_sync_service.h" -#include "olap/txn_manager.h" #include "olap/task/engine_task.h" -#include "olap/rowset/rowset_id_generator.h" -#include "olap/fs/fs_util.h" +#include "olap/txn_manager.h" #include "runtime/heartbeat_flags.h" #include "util/countdown_latch.h" #include "util/thread.h" #include "util/threadpool.h" - namespace doris { class DataDir; @@ -251,7 +250,7 @@ class StorageEngine { // 重新加载数据。 void _start_disk_stat_monitor(); - Status _compaction_tasks_producer_callback(); + void _compaction_tasks_producer_callback(); vector _compaction_tasks_generator(CompactionType compaction_type, std::vector data_dirs); private: @@ -349,13 +348,11 @@ class StorageEngine { HeartbeatFlags* _heartbeat_flags; - DISALLOW_COPY_AND_ASSIGN(StorageEngine); - - std::map _map_disk_compaction_num; - std::unique_ptr _thread_pool_compaction; + std::unique_ptr _compaction_thread_pool; CompactionPermitLimiter _permit_limiter; - std::mutex _mutex_compaction_metrics; + + DISALLOW_COPY_AND_ASSIGN(StorageEngine); }; } // namespace doris diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index ea841642c0504b..a0b1263bc305b0 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -741,6 +741,16 @@ bool Tablet::can_do_compaction() { return true; } +const uint32_t Tablet::calc_compaction_score(CompactionType compaction_type) const { + uint32_t permits; + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + permits = calc_cumulative_compaction_score(); + } else { + permits = calc_base_compaction_score(); + } + return permits; +} + const uint32_t Tablet::calc_cumulative_compaction_score() const { uint32_t score = 0; _cumulative_compaction_policy->calc_cumulative_compaction_score( diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index c32daad3b51008..ddf2248e6ebfb1 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -163,6 +163,7 @@ class Tablet : public BaseTablet { // operation for compaction bool can_do_compaction(); + const uint32_t calc_compaction_score(CompactionType compaction_type) const; const uint32_t calc_cumulative_compaction_score() const; const uint32_t calc_base_compaction_score() const; static void compute_version_hash_from_rowsets(const std::vector& rowsets, From b7dd382de92f2f6c411eb7208f46644927122770 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Thu, 24 Sep 2020 20:33:11 +0800 Subject: [PATCH 19/27] modify metrics --- be/src/common/config.h | 8 +-- be/src/olap/base_compaction.cpp | 3 +- be/src/olap/compaction.cpp | 13 ++++- be/src/olap/compaction.h | 2 +- be/src/olap/compaction_permit_limiter.cpp | 5 +- be/src/olap/compaction_permit_limiter.h | 6 +-- be/src/olap/cumulative_compaction.cpp | 3 +- be/src/olap/data_dir.cpp | 16 +++--- be/src/olap/data_dir.h | 5 +- be/src/olap/olap_server.cpp | 66 ++++++----------------- be/src/olap/storage_engine.h | 4 -- be/src/olap/tablet.cpp | 7 ++- 12 files changed, 53 insertions(+), 85 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index 55d80a74ffe26f..2daf9a4d7cb933 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -291,7 +291,7 @@ namespace config { // cumulative compaction policy: max delta file's size unit:B CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5"); - CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000"); + CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "5"); //1000 CONF_mInt64(cumulative_compaction_budgeted_bytes, "104857600"); // CONF_Int32(cumulative_compaction_write_mbytes_per_sec, "100"); // cumulative compaction skips recently published deltas in order to prevent @@ -314,10 +314,10 @@ namespace config { CONF_mBool(enable_over_sold, "true"); // Sleep time of compaction tasks producer thread after each tasks generation, in seconds. - CONF_mInt32(generate_compaction_tasks_interval_seconds, "5") + CONF_mInt32(generate_compaction_tasks_interval_seconds, "1") - // Compaction task number for per disk. - CONF_mInt32(compaction_task_num_per_disk, "5"); + // Compaction task number per disk. + CONF_mInt32(compaction_task_num_per_disk, "10"); // How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation. CONF_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round, "9"); diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp index bb2404ca3be0b8..a38c244f478daf 100644 --- a/be/src/olap/base_compaction.cpp +++ b/be/src/olap/base_compaction.cpp @@ -45,7 +45,8 @@ OLAPStatus BaseCompaction::compact() { TRACE_COUNTER_INCREMENT("input_rowsets_count", _input_rowsets.size()); // 2. do base compaction, merge rowsets - RETURN_NOT_OK(do_compaction()); + int64_t permits = _tablet->calc_base_compaction_score(); + RETURN_NOT_OK(do_compaction(permits)); TRACE("compaction finished"); // 3. set state to success diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index d0e5bc88da1383..80a402b7c55de9 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -35,9 +35,18 @@ Compaction::Compaction(TabletSharedPtr tablet, const std::string& label, const s Compaction::~Compaction() {} -OLAPStatus Compaction::do_compaction() { - TRACE("got concurrency lock and start to do compaction"); +OLAPStatus Compaction::do_compaction(int64_t permits) { + TRACE("start to do compaction"); + _tablet->data_dir()->disks_compaction_score_increment(permits); + _tablet->data_dir()->disks_compaction_num_increment(1); + DorisMetrics::instance()->total_compaction_score->increment(permits); + DorisMetrics::instance()->total_compaction_num->increment(1); OLAPStatus st = do_compaction_impl(); + sleep(60); + _tablet->data_dir()->disks_compaction_score_increment(-permits); + _tablet->data_dir()->disks_compaction_num_increment(-1); + DorisMetrics::instance()->total_compaction_score->increment(-permits); + DorisMetrics::instance()->total_compaction_num->increment(-1); return st; } diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h index 45735937f5adc2..20a567cdc82bed 100644 --- a/be/src/olap/compaction.h +++ b/be/src/olap/compaction.h @@ -54,7 +54,7 @@ class Compaction { virtual std::string compaction_name() const = 0; virtual ReaderType compaction_type() const = 0; - OLAPStatus do_compaction(); + OLAPStatus do_compaction(int64_t permits); OLAPStatus do_compaction_impl(); void modify_rowsets(); diff --git a/be/src/olap/compaction_permit_limiter.cpp b/be/src/olap/compaction_permit_limiter.cpp index 3fd40a2f3033c7..3f86f700d3f83c 100644 --- a/be/src/olap/compaction_permit_limiter.cpp +++ b/be/src/olap/compaction_permit_limiter.cpp @@ -21,7 +21,7 @@ namespace doris { CompactionPermitLimiter::CompactionPermitLimiter() : _used_permits(0) {} -bool CompactionPermitLimiter::request(uint32_t permits) { +bool CompactionPermitLimiter::request(int64_t permits) { if (permits > config::total_permits_for_compaction_score) { if (config::enable_over_sold) { std::unique_lock lock(_over_sold_mutex); @@ -43,7 +43,8 @@ bool CompactionPermitLimiter::request(uint32_t permits) { } } -void CompactionPermitLimiter::release(uint32_t permits) { +void CompactionPermitLimiter::release(int64_t permits) { _used_permits -= permits; + _cv.notify_one(); } } // namespace doris diff --git a/be/src/olap/compaction_permit_limiter.h b/be/src/olap/compaction_permit_limiter.h index 8e225a8bf04e71..78c1143b01410d 100644 --- a/be/src/olap/compaction_permit_limiter.h +++ b/be/src/olap/compaction_permit_limiter.h @@ -36,12 +36,12 @@ class CompactionPermitLimiter { CompactionPermitLimiter(); virtual ~CompactionPermitLimiter() {} - bool request(uint32_t permits); + bool request(int64_t permits); - void release(uint32_t permits); + void release(int64_t permits); private: - // sum of "permites" held by executing compaction tasks currently + // sum of "permits" held by executing compaction tasks currently AtomicInt64 _used_permits; std::mutex _over_sold_mutex; std::condition_variable _cv; diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp index 6611e5102be336..c2c8755c694e6e 100755 --- a/be/src/olap/cumulative_compaction.cpp +++ b/be/src/olap/cumulative_compaction.cpp @@ -53,7 +53,8 @@ OLAPStatus CumulativeCompaction::compact() { TRACE_COUNTER_INCREMENT("input_rowsets_count", _input_rowsets.size()); // 3. do cumulative compaction, merge rowsets - RETURN_NOT_OK(do_compaction()); + int64_t permits = _tablet->calc_cumulative_compaction_score(); + RETURN_NOT_OK(do_compaction(permits)); TRACE("compaction finished"); // 4. set state to success diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index c88cd3b60e4f13..f0574b6177255b 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -1001,19 +1001,15 @@ bool DataDir::reach_capacity_limit(int64_t incoming_data_size) { return false; } -void DataDir::update_disks_compaction_score(int64_t compaction_score) { - disks_compaction_score->set_value(compaction_score); -} - -int64_t DataDir::get_disks_compaction_score() { - return disks_compaction_score->value(); -} - -void DataDir::update_disks_compaction_num(int64_t compaction_num) { - disks_compaction_num->set_value(compaction_num); +void DataDir::disks_compaction_score_increment(int64_t delta) { + disks_compaction_score->increment(delta); } int64_t DataDir::get_disks_compaction_num() { return disks_compaction_num->value(); } + +void DataDir::disks_compaction_num_increment(int64_t delta) { + disks_compaction_num->increment(delta); +} } // namespace doris diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h index 57bcdac39ec537..6a2097017cdc95 100644 --- a/be/src/olap/data_dir.h +++ b/be/src/olap/data_dir.h @@ -128,11 +128,10 @@ class DataDir { std::set tablet_set() { return _tablet_set; } - void update_disks_compaction_score(int64_t compaction_score); - int64_t get_disks_compaction_score(); + void disks_compaction_score_increment(int64_t delta); - void update_disks_compaction_num(int64_t compaction_num); int64_t get_disks_compaction_num(); + void disks_compaction_num_increment(int64_t delta); private: std::string _cluster_id_path() const { return _path + CLUSTER_ID_PREFIX; } diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 87f11ceadbd857..cb53787dabadc6 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -75,13 +75,15 @@ Status StorageEngine::start_bg_threads() { int32_t max_thread_num = config::max_compaction_threads; int32_t min_thread_num = config::min_compaction_threads; - ThreadPoolBuilder("CompactionTaskThreadPool").set_min_threads(min_thread_num).set_max_threads(max_thread_num).build(&_compaction_thread_pool); + ThreadPoolBuilder("CompactionTaskThreadPool") + .set_min_threads(min_thread_num) + .set_max_threads(max_thread_num) + .build(&_compaction_thread_pool); // compaction tasks producer thread - RETURN_IF_ERROR( - Thread::create("StorageEngine", "compaction_tasks_producer_thread", - [this]() { this->_compaction_tasks_producer_callback(); }, - &_compaction_tasks_producer_thread)); + RETURN_IF_ERROR(Thread::create("StorageEngine", "compaction_tasks_producer_thread", + [this]() { this->_compaction_tasks_producer_callback(); }, + &_compaction_tasks_producer_thread)); LOG(INFO) << "compaction tasks producer thread started"; // tablet checkpoint thread @@ -143,26 +145,6 @@ void StorageEngine::_fd_cache_clean_callback() { } } -void StorageEngine::_base_compaction_task(TabletSharedPtr tablet, uint32_t permits) { - tablet->data_dir()->update_disks_compaction_score( - tablet->data_dir()->get_disks_compaction_score() + permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + - 1); - DorisMetrics::instance()->total_compaction_score->set_value( - DorisMetrics::instance()->total_compaction_score->value() + permits); - DorisMetrics::instance()->total_compaction_num->increment(1); - _perform_base_compaction(tablet); - sleep(25); - _permit_limiter.release(permits); - tablet->data_dir()->update_disks_compaction_score( - tablet->data_dir()->get_disks_compaction_score() - permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - - 1); - DorisMetrics::instance()->total_compaction_score->set_value( - DorisMetrics::instance()->total_compaction_score->value() + permits); - DorisMetrics::instance()->total_compaction_num->increment(-1); -} - void StorageEngine::_garbage_sweeper_thread_callback() { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); @@ -253,26 +235,6 @@ void StorageEngine::_check_cumulative_compaction_config() { } } -void StorageEngine::_cumulative_compaction_task(TabletSharedPtr tablet, uint32_t permits) { - tablet->data_dir()->update_disks_compaction_score( - tablet->data_dir()->get_disks_compaction_score() + permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() + - 1); - DorisMetrics::instance()->total_compaction_score->set_value( - DorisMetrics::instance()->total_compaction_score->value() + permits); - DorisMetrics::instance()->total_compaction_num->increment(1); - _perform_cumulative_compaction(tablet); - sleep(25); - _permit_limiter.release(permits); - tablet->data_dir()->update_disks_compaction_score( - tablet->data_dir()->get_disks_compaction_score() - permits); - tablet->data_dir()->update_disks_compaction_num(tablet->data_dir()->get_disks_compaction_num() - - 1); - DorisMetrics::instance()->total_compaction_score->set_value( - DorisMetrics::instance()->total_compaction_score->value() - permits); - DorisMetrics::instance()->total_compaction_num->increment(-1); -} - void StorageEngine::_unused_rowset_monitor_thread_callback() { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); @@ -378,15 +340,17 @@ void StorageEngine::_compaction_tasks_producer_callback() { for (const auto& tablet : tablets_compaction) { if (tablet->data_dir()->get_disks_compaction_num() < config::compaction_task_num_per_disk) { - uint32_t permits = tablet->calc_compaction_score(compaction_type); + int64_t permits = tablet->calc_compaction_score(compaction_type); if (_permit_limiter.request(permits)) { if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { _compaction_thread_pool->submit_func([this, tablet, permits]() { - this->_cumulative_compaction_task(tablet, permits); + this->_perform_cumulative_compaction(tablet); + this->_permit_limiter.release(permits); }); } else { _compaction_thread_pool->submit_func([this, tablet, permits]() { - this->_base_compaction_task(tablet, permits); + this->_perform_base_compaction(tablet); + this->_permit_limiter.release(permits); }); } } @@ -404,9 +368,11 @@ vector StorageEngine::_compaction_tasks_generator( if (!data_dir->reach_capacity_limit(0)) { TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction(compaction_type, data_dir); - tablets_compaction.emplace_back(tablet); + if (tablet != nullptr) { + tablets_compaction.emplace_back(tablet); + } } } return tablets_compaction; } -} // namespace doris +} // namespace doris diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 67acd1610272a2..95ecdd68d68030 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -213,12 +213,8 @@ class StorageEngine { // unused rowset monitor thread void _unused_rowset_monitor_thread_callback(); - // base compaction thread process function - void _base_compaction_task(TabletSharedPtr tablet, uint32_t permits); // check cumulative compaction config void _check_cumulative_compaction_config(); - // cumulative process function - void _cumulative_compaction_task(TabletSharedPtr tablet, uint32_t permits); // garbage sweep thread process function. clear snapshot and trash folder void _garbage_sweeper_thread_callback(); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index a0b1263bc305b0..f2780522fd44c8 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -742,13 +742,12 @@ bool Tablet::can_do_compaction() { } const uint32_t Tablet::calc_compaction_score(CompactionType compaction_type) const { - uint32_t permits; if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - permits = calc_cumulative_compaction_score(); + return calc_cumulative_compaction_score(); } else { - permits = calc_base_compaction_score(); + DCHECK_EQ(compaction_type, CompactionType::BASE_COMPACTION); + return calc_base_compaction_score(); } - return permits; } const uint32_t Tablet::calc_cumulative_compaction_score() const { From 256d25851ca69bc0d7f5ec38f37ce6861f7df94e Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Thu, 24 Sep 2020 21:47:48 +0800 Subject: [PATCH 20/27] modify metrics --- be/src/common/config.h | 12 ++++++--- be/src/olap/compaction.cpp | 1 - be/src/olap/olap_server.cpp | 54 ++++++++++++++++++++----------------- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index 2daf9a4d7cb933..96525d20deb1cc 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -261,6 +261,10 @@ namespace config { // whether to disable page cache feature in storage CONF_Bool(disable_storage_page_cache, "false"); + // be policy + // whether disable automatic compaction task + CONF_mBool(disable_auto_compaction, "false"); + // CONF_Int64(base_compaction_start_hour, "20"); // CONF_Int64(base_compaction_end_hour, "7"); CONF_mInt64(base_compaction_num_cumulative_deltas, "5"); @@ -291,7 +295,7 @@ namespace config { // cumulative compaction policy: max delta file's size unit:B CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5"); - CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "5"); //1000 + CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000"); CONF_mInt64(cumulative_compaction_budgeted_bytes, "104857600"); // CONF_Int32(cumulative_compaction_write_mbytes_per_sec, "100"); // cumulative compaction skips recently published deltas in order to prevent @@ -308,16 +312,16 @@ namespace config { CONF_mInt32(max_compaction_threads, "10"); // The upper limit of "permites" held by all compaction tasks. This config can be set to limit memory consumption for compaction. - CONF_mInt64(total_permits_for_compaction_score, "15000") + CONF_mInt64(total_permits_for_compaction_score, "10000") // Whether compaction task is allowed to start when compaction score of current tablet is out of upper limit. CONF_mBool(enable_over_sold, "true"); // Sleep time of compaction tasks producer thread after each tasks generation, in seconds. - CONF_mInt32(generate_compaction_tasks_interval_seconds, "1") + CONF_mInt32(generate_compaction_tasks_interval_seconds, "2") // Compaction task number per disk. - CONF_mInt32(compaction_task_num_per_disk, "10"); + CONF_mInt32(compaction_task_num_per_disk, "2"); // How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation. CONF_mInt32(cumulative_compaction_rounds_for_each_base_compaction_round, "9"); diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 80a402b7c55de9..e06355797f768a 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -42,7 +42,6 @@ OLAPStatus Compaction::do_compaction(int64_t permits) { DorisMetrics::instance()->total_compaction_score->increment(permits); DorisMetrics::instance()->total_compaction_num->increment(1); OLAPStatus st = do_compaction_impl(); - sleep(60); _tablet->data_dir()->disks_compaction_score_increment(-permits); _tablet->data_dir()->disks_compaction_num_increment(-1); DorisMetrics::instance()->total_compaction_score->increment(-permits); diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index cb53787dabadc6..6db4dcef0fe7e9 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -27,6 +27,7 @@ #include #include +#include "agent/cgroups_mgr.h" #include "common/status.h" #include "olap/cumulative_compaction.h" #include "olap/olap_common.h" @@ -327,31 +328,34 @@ void StorageEngine::_compaction_tasks_producer_callback() { int round = 0; CompactionType compaction_type; do { - if (round < config::cumulative_compaction_rounds_for_each_base_compaction_round) { - compaction_type = CompactionType::CUMULATIVE_COMPACTION; - round++; - } else { - compaction_type = CompactionType::BASE_COMPACTION; - round = 0; - } - LOG(INFO) << "try to generate a batch of compaction tasks!"; - vector tablets_compaction = - _compaction_tasks_generator(compaction_type, data_dirs); - for (const auto& tablet : tablets_compaction) { - if (tablet->data_dir()->get_disks_compaction_num() < - config::compaction_task_num_per_disk) { - int64_t permits = tablet->calc_compaction_score(compaction_type); - if (_permit_limiter.request(permits)) { - if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - _compaction_thread_pool->submit_func([this, tablet, permits]() { - this->_perform_cumulative_compaction(tablet); - this->_permit_limiter.release(permits); - }); - } else { - _compaction_thread_pool->submit_func([this, tablet, permits]() { - this->_perform_base_compaction(tablet); - this->_permit_limiter.release(permits); - }); + if (!config::disable_auto_compaction) { + CgroupsMgr::apply_system_cgroup(); + if (round < config::cumulative_compaction_rounds_for_each_base_compaction_round) { + compaction_type = CompactionType::CUMULATIVE_COMPACTION; + round++; + } else { + compaction_type = CompactionType::BASE_COMPACTION; + round = 0; + } + LOG(INFO) << "try to generate a batch of compaction tasks!"; + vector tablets_compaction = + _compaction_tasks_generator(compaction_type, data_dirs); + for (const auto &tablet : tablets_compaction) { + if (tablet->data_dir()->get_disks_compaction_num() < + config::compaction_task_num_per_disk) { + int64_t permits = tablet->calc_compaction_score(compaction_type); + if (_permit_limiter.request(permits)) { + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + _compaction_thread_pool->submit_func([this, tablet, permits]() { + this->_perform_cumulative_compaction(tablet); + this->_permit_limiter.release(permits); + }); + } else { + _compaction_thread_pool->submit_func([this, tablet, permits]() { + this->_perform_base_compaction(tablet); + this->_permit_limiter.release(permits); + }); + } } } } From 9484f52d6ade61188f34aeee8717d236de467350 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Thu, 24 Sep 2020 22:10:54 +0800 Subject: [PATCH 21/27] modify metrics --- be/src/common/config.h | 2 +- be/src/olap/olap_server.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index 96525d20deb1cc..eb3bb106459d0e 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -311,7 +311,7 @@ namespace config { CONF_mInt32(min_compaction_threads, "10"); CONF_mInt32(max_compaction_threads, "10"); - // The upper limit of "permites" held by all compaction tasks. This config can be set to limit memory consumption for compaction. + // The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction. CONF_mInt64(total_permits_for_compaction_score, "10000") // Whether compaction task is allowed to start when compaction score of current tablet is out of upper limit. diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 6db4dcef0fe7e9..26aa0906376eaa 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -338,9 +338,9 @@ void StorageEngine::_compaction_tasks_producer_callback() { round = 0; } LOG(INFO) << "try to generate a batch of compaction tasks!"; - vector tablets_compaction = + vector tablets_compaction = _compaction_tasks_generator(compaction_type, data_dirs); - for (const auto &tablet : tablets_compaction) { + for (const auto& tablet : tablets_compaction) { if (tablet->data_dir()->get_disks_compaction_num() < config::compaction_task_num_per_disk) { int64_t permits = tablet->calc_compaction_score(compaction_type); From 18c2933b654f25a878d5ad75e172289bf82b3689 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Fri, 25 Sep 2020 11:24:40 +0800 Subject: [PATCH 22/27] modify metrics --- be/src/olap/compaction.cpp | 4 ---- be/src/olap/olap_server.cpp | 4 ++-- be/src/util/doris_metrics.cpp | 6 ------ be/src/util/doris_metrics.h | 3 --- 4 files changed, 2 insertions(+), 15 deletions(-) diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index e06355797f768a..f81a029cc1a56b 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -39,13 +39,9 @@ OLAPStatus Compaction::do_compaction(int64_t permits) { TRACE("start to do compaction"); _tablet->data_dir()->disks_compaction_score_increment(permits); _tablet->data_dir()->disks_compaction_num_increment(1); - DorisMetrics::instance()->total_compaction_score->increment(permits); - DorisMetrics::instance()->total_compaction_num->increment(1); OLAPStatus st = do_compaction_impl(); _tablet->data_dir()->disks_compaction_score_increment(-permits); _tablet->data_dir()->disks_compaction_num_increment(-1); - DorisMetrics::instance()->total_compaction_score->increment(-permits); - DorisMetrics::instance()->total_compaction_num->increment(-1); return st; } diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 26aa0906376eaa..5cebb258ea6353 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -33,7 +33,6 @@ #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/storage_engine.h" -#include "util/doris_metrics.h" #include "util/time.h" using std::string; @@ -329,7 +328,6 @@ void StorageEngine::_compaction_tasks_producer_callback() { CompactionType compaction_type; do { if (!config::disable_auto_compaction) { - CgroupsMgr::apply_system_cgroup(); if (round < config::cumulative_compaction_rounds_for_each_base_compaction_round) { compaction_type = CompactionType::CUMULATIVE_COMPACTION; round++; @@ -347,11 +345,13 @@ void StorageEngine::_compaction_tasks_producer_callback() { if (_permit_limiter.request(permits)) { if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { _compaction_thread_pool->submit_func([this, tablet, permits]() { + CgroupsMgr::apply_system_cgroup(); this->_perform_cumulative_compaction(tablet); this->_permit_limiter.release(permits); }); } else { _compaction_thread_pool->submit_func([this, tablet, permits]() { + CgroupsMgr::apply_system_cgroup(); this->_perform_base_compaction(tablet); this->_permit_limiter.release(permits); }); diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp index beae32a27e7d7f..023074a66e9d0f 100644 --- a/be/src/util/doris_metrics.cpp +++ b/be/src/util/doris_metrics.cpp @@ -110,9 +110,6 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(process_fd_num_limit_hard, MetricUnit::NOUNIT DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(tablet_cumulative_max_compaction_score, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(tablet_base_max_compaction_score, MetricUnit::NOUNIT); -DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(total_compaction_score, MetricUnit::NOUNIT); -DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(total_compaction_num, MetricUnit::NOUNIT); - DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(push_request_write_bytes_per_second, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(query_scan_bytes_per_second, MetricUnit::BYTES); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(max_disk_io_util_percent, MetricUnit::PERCENT); @@ -213,9 +210,6 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_cumulative_max_compaction_score); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_base_max_compaction_score); - INT_GAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_score); - INT_GAUGE_METRIC_REGISTER(_server_metric_entity, total_compaction_num); - INT_GAUGE_METRIC_REGISTER(_server_metric_entity, push_request_write_bytes_per_second); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, query_scan_bytes_per_second); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, max_disk_io_util_percent); diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h index 5d357908786e78..3ffbe746d88203 100644 --- a/be/src/util/doris_metrics.h +++ b/be/src/util/doris_metrics.h @@ -133,9 +133,6 @@ class DorisMetrics { IntGauge* tablet_cumulative_max_compaction_score; IntGauge* tablet_base_max_compaction_score; - IntGauge* total_compaction_score; - IntGauge* total_compaction_num; - // The following metrics will be calculated // by metric calculator IntGauge* push_request_write_bytes_per_second; From 5c6879b0d7723e7b0daf2dbd45bffa1e2009b323 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Sun, 27 Sep 2020 22:16:19 +0800 Subject: [PATCH 23/27] optimize compaction tasks producer --- be/src/common/config.h | 9 ++- be/src/olap/compaction_permit_limiter.cpp | 2 +- be/src/olap/data_dir.cpp | 4 -- be/src/olap/data_dir.h | 1 - be/src/olap/olap_server.cpp | 73 ++++++++++++++++------- be/src/olap/storage_engine.h | 5 ++ be/src/olap/tablet_manager.cpp | 12 +++- be/src/olap/tablet_manager.h | 4 +- 8 files changed, 72 insertions(+), 38 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index eb3bb106459d0e..8f10dd06416ead 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -264,6 +264,8 @@ namespace config { // be policy // whether disable automatic compaction task CONF_mBool(disable_auto_compaction, "false"); + // check the configuration of auto compaction in seconds when auto compaction disabled + CONF_mInt32(check_auto_compaction_interval_seconds, "5"); // CONF_Int64(base_compaction_start_hour, "20"); // CONF_Int64(base_compaction_end_hour, "7"); @@ -312,13 +314,10 @@ namespace config { CONF_mInt32(max_compaction_threads, "10"); // The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction. - CONF_mInt64(total_permits_for_compaction_score, "10000") + CONF_mInt64(total_permits_for_compaction_score, "10000"); // Whether compaction task is allowed to start when compaction score of current tablet is out of upper limit. - CONF_mBool(enable_over_sold, "true"); - - // Sleep time of compaction tasks producer thread after each tasks generation, in seconds. - CONF_mInt32(generate_compaction_tasks_interval_seconds, "2") + CONF_mBool(enable_compaction_permit_over_sold, "true"); // Compaction task number per disk. CONF_mInt32(compaction_task_num_per_disk, "2"); diff --git a/be/src/olap/compaction_permit_limiter.cpp b/be/src/olap/compaction_permit_limiter.cpp index 3f86f700d3f83c..7e1067cdfae5fd 100644 --- a/be/src/olap/compaction_permit_limiter.cpp +++ b/be/src/olap/compaction_permit_limiter.cpp @@ -23,7 +23,7 @@ CompactionPermitLimiter::CompactionPermitLimiter() : _used_permits(0) {} bool CompactionPermitLimiter::request(int64_t permits) { if (permits > config::total_permits_for_compaction_score) { - if (config::enable_over_sold) { + if (config::enable_compaction_permit_over_sold) { std::unique_lock lock(_over_sold_mutex); _cv.wait(lock, [=] { return _used_permits == 0 || diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index f0574b6177255b..c20849b05fa598 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -1005,10 +1005,6 @@ void DataDir::disks_compaction_score_increment(int64_t delta) { disks_compaction_score->increment(delta); } -int64_t DataDir::get_disks_compaction_num() { - return disks_compaction_num->value(); -} - void DataDir::disks_compaction_num_increment(int64_t delta) { disks_compaction_num->increment(delta); } diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h index 6a2097017cdc95..898f644f122581 100644 --- a/be/src/olap/data_dir.h +++ b/be/src/olap/data_dir.h @@ -130,7 +130,6 @@ class DataDir { void disks_compaction_score_increment(int64_t delta); - int64_t get_disks_compaction_num(); void disks_compaction_num_increment(int64_t delta); private: diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 5cebb258ea6353..9a6a5afe580ee7 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -319,14 +319,16 @@ void StorageEngine::_compaction_tasks_producer_callback() { #endif LOG(INFO) << "try to start compaction producer process!"; + std::vector tablet_submitted; std::vector data_dirs; for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); + _tablet_submitted_compaction[tmp_store.second] = tablet_submitted; } int round = 0; CompactionType compaction_type; - do { + while (true) { if (!config::disable_auto_compaction) { if (round < config::cumulative_compaction_rounds_for_each_base_compaction_round) { compaction_type = CompactionType::CUMULATIVE_COMPACTION; @@ -335,33 +337,51 @@ void StorageEngine::_compaction_tasks_producer_callback() { compaction_type = CompactionType::BASE_COMPACTION; round = 0; } - LOG(INFO) << "try to generate a batch of compaction tasks!"; vector tablets_compaction = _compaction_tasks_generator(compaction_type, data_dirs); for (const auto& tablet : tablets_compaction) { - if (tablet->data_dir()->get_disks_compaction_num() < - config::compaction_task_num_per_disk) { - int64_t permits = tablet->calc_compaction_score(compaction_type); - if (_permit_limiter.request(permits)) { - if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - _compaction_thread_pool->submit_func([this, tablet, permits]() { - CgroupsMgr::apply_system_cgroup(); - this->_perform_cumulative_compaction(tablet); - this->_permit_limiter.release(permits); - }); - } else { - _compaction_thread_pool->submit_func([this, tablet, permits]() { - CgroupsMgr::apply_system_cgroup(); - this->_perform_base_compaction(tablet); - this->_permit_limiter.release(permits); - }); + int64_t permits = tablet->calc_compaction_score(compaction_type); + if (!_permit_limiter.request(permits)) { + std::unique_lock lock(_wait_permits_mutex); + _wait_permits_cv.wait(lock, + [=] { return _permit_limiter.request(permits); }); + } + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + _compaction_thread_pool->submit_func([this, tablet, permits]() { + CgroupsMgr::apply_system_cgroup(); + this->_perform_cumulative_compaction(tablet); + this->_permit_limiter.release(permits); + _wait_permits_cv.notify_one(); + std::unique_lock lock(_tablet_submitted_compaction_mutex); + vector::iterator it_tablet = find( + _tablet_submitted_compaction[tablet->data_dir()].begin(), + _tablet_submitted_compaction[tablet->data_dir()].end(), tablet); + if (it_tablet != _tablet_submitted_compaction[tablet->data_dir()].end()) { + _tablet_submitted_compaction[tablet->data_dir()].erase(it_tablet); } - } + }); + } else { + _compaction_thread_pool->submit_func([this, tablet, permits]() { + CgroupsMgr::apply_system_cgroup(); + this->_perform_base_compaction(tablet); + this->_permit_limiter.release(permits); + _wait_permits_cv.notify_one(); + std::unique_lock lock(_tablet_submitted_compaction_mutex); + vector::iterator it_tablet = find( + _tablet_submitted_compaction[tablet->data_dir()].begin(), + _tablet_submitted_compaction[tablet->data_dir()].end(), tablet); + if (it_tablet != _tablet_submitted_compaction[tablet->data_dir()].end()) { + _tablet_submitted_compaction[tablet->data_dir()].erase(it_tablet); + } + }); } + std::unique_lock lock(_tablet_submitted_compaction_mutex); + _tablet_submitted_compaction[tablet->data_dir()].emplace_back(tablet); } + } else { + sleep(config::check_auto_compaction_interval_seconds); } - } while (!_stop_background_threads_latch.wait_for( - MonoDelta::FromSeconds(config::generate_compaction_tasks_interval_seconds))); + } } vector StorageEngine::_compaction_tasks_generator( @@ -369,9 +389,16 @@ vector StorageEngine::_compaction_tasks_generator( vector tablets_compaction; std::random_shuffle(data_dirs.begin(), data_dirs.end()); for (auto data_dir : data_dirs) { + if (_tablet_submitted_compaction[data_dir].size() >= config::compaction_task_num_per_disk) { + continue; + } if (!data_dir->reach_capacity_limit(0)) { - TabletSharedPtr tablet = - _tablet_manager->find_best_tablet_to_compaction(compaction_type, data_dir); + TabletSharedPtr tablet; + { + std::unique_lock lock(_tablet_submitted_compaction_mutex); + tablet = _tablet_manager->find_best_tablet_to_compaction( + compaction_type, data_dir, _tablet_submitted_compaction[data_dir]); + } if (tablet != nullptr) { tablets_compaction.emplace_back(tablet); } diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 95ecdd68d68030..600e90ac8e2338 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -347,6 +347,11 @@ class StorageEngine { std::unique_ptr _compaction_thread_pool; CompactionPermitLimiter _permit_limiter; + std::map> _tablet_submitted_compaction; + std::mutex _tablet_submitted_compaction_mutex; + + std::mutex _wait_permits_mutex; + std::condition_variable _wait_permits_cv; DISALLOW_COPY_AND_ASSIGN(StorageEngine); }; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 6260f2163184e3..c034516bc7558d 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -679,8 +679,9 @@ void TabletManager::get_tablet_stat(TTabletStatResult* result) { result->__set_tablets_stats(_tablet_stat_cache); } -TabletSharedPtr TabletManager::find_best_tablet_to_compaction(CompactionType compaction_type, - DataDir* data_dir) { +TabletSharedPtr TabletManager::find_best_tablet_to_compaction( + CompactionType compaction_type, DataDir* data_dir, + vector& tablet_submitted_compaction) { int64_t now_ms = UnixMillis(); const string& compaction_type_str = compaction_type == CompactionType::BASE_COMPACTION ? "base" : "cumulative"; uint32_t highest_score = 0; @@ -690,6 +691,12 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(CompactionType com tablet_map_t& tablet_map = _tablet_map_array[i]; for (tablet_map_t::value_type& table_ins : tablet_map){ for (TabletSharedPtr& tablet_ptr : table_ins.second.table_arr) { + vector::iterator it_tablet = + find(tablet_submitted_compaction.begin(), tablet_submitted_compaction.end(), + tablet_ptr); + if (it_tablet != tablet_submitted_compaction.end()) { + continue; + } AlterTabletTaskSharedPtr cur_alter_task = tablet_ptr->alter_task(); if (cur_alter_task != nullptr && cur_alter_task->alter_state() != ALTER_FINISHED @@ -738,7 +745,6 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(CompactionType com } } - uint32_t table_score = 0; { ReadLock rdlock(tablet_ptr->get_header_lock_ptr()); diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index ea4de8c69e96c7..6f9c0f80c295cd 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -69,7 +69,9 @@ class TabletManager { OLAPStatus drop_tablets_on_error_root_path(const std::vector& tablet_info_vec); - TabletSharedPtr find_best_tablet_to_compaction(CompactionType compaction_type, DataDir* data_dir); + TabletSharedPtr find_best_tablet_to_compaction( + CompactionType compaction_type, DataDir* data_dir, + vector& tablet_submitted_compaction); TabletSharedPtr get_tablet(TTabletId tablet_id, SchemaHash schema_hash, bool include_deleted = false, std::string* err = nullptr); From 7c343f47cff3267eb364be9bf6bb72f53ccdb63d Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Mon, 28 Sep 2020 10:19:22 +0800 Subject: [PATCH 24/27] modify compaction tasks producer --- be/src/olap/compaction_permit_limiter.cpp | 12 ++-- be/src/olap/compaction_permit_limiter.h | 4 +- be/src/olap/olap_server.cpp | 76 +++++++++++------------ be/src/olap/storage_engine.h | 3 - 4 files changed, 46 insertions(+), 49 deletions(-) diff --git a/be/src/olap/compaction_permit_limiter.cpp b/be/src/olap/compaction_permit_limiter.cpp index 7e1067cdfae5fd..39f3add4e56992 100644 --- a/be/src/olap/compaction_permit_limiter.cpp +++ b/be/src/olap/compaction_permit_limiter.cpp @@ -25,9 +25,9 @@ bool CompactionPermitLimiter::request(int64_t permits) { if (permits > config::total_permits_for_compaction_score) { if (config::enable_compaction_permit_over_sold) { std::unique_lock lock(_over_sold_mutex); - _cv.wait(lock, [=] { + _over_sold_cv.wait(lock, [=] { return _used_permits == 0 || - _used_permits + permits < config::total_permits_for_compaction_score; + _used_permits + permits <= config::total_permits_for_compaction_score; }); _used_permits += permits; return true; @@ -36,7 +36,10 @@ bool CompactionPermitLimiter::request(int64_t permits) { } } else { if (_used_permits + permits > config::total_permits_for_compaction_score) { - return false; + std::unique_lock lock(_wait_permits_mutex); + _wait_permits_cv.wait(lock, [=] { + return _used_permits + permits <= config::total_permits_for_compaction_score; + }); } _used_permits += permits; return true; @@ -45,6 +48,7 @@ bool CompactionPermitLimiter::request(int64_t permits) { void CompactionPermitLimiter::release(int64_t permits) { _used_permits -= permits; - _cv.notify_one(); + _over_sold_cv.notify_one(); + _wait_permits_cv.notify_one(); } } // namespace doris diff --git a/be/src/olap/compaction_permit_limiter.h b/be/src/olap/compaction_permit_limiter.h index 78c1143b01410d..0c4c4e2d0a4369 100644 --- a/be/src/olap/compaction_permit_limiter.h +++ b/be/src/olap/compaction_permit_limiter.h @@ -44,6 +44,8 @@ class CompactionPermitLimiter { // sum of "permits" held by executing compaction tasks currently AtomicInt64 _used_permits; std::mutex _over_sold_mutex; - std::condition_variable _cv; + std::condition_variable _over_sold_cv; + std::mutex _wait_permits_mutex; + std::condition_variable _wait_permits_cv; }; } // namespace doris diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 9a6a5afe580ee7..903b9f002df96b 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -341,42 +341,39 @@ void StorageEngine::_compaction_tasks_producer_callback() { _compaction_tasks_generator(compaction_type, data_dirs); for (const auto& tablet : tablets_compaction) { int64_t permits = tablet->calc_compaction_score(compaction_type); - if (!_permit_limiter.request(permits)) { - std::unique_lock lock(_wait_permits_mutex); - _wait_permits_cv.wait(lock, - [=] { return _permit_limiter.request(permits); }); + if (_permit_limiter.request(permits)) { + if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { + _compaction_thread_pool->submit_func([this, tablet, permits]() { + CgroupsMgr::apply_system_cgroup(); + _perform_cumulative_compaction(tablet); + _permit_limiter.release(permits); + std::unique_lock lock(_tablet_submitted_compaction_mutex); + vector::iterator it_tablet = find( + _tablet_submitted_compaction[tablet->data_dir()].begin(), + _tablet_submitted_compaction[tablet->data_dir()].end(), tablet); + if (it_tablet != + _tablet_submitted_compaction[tablet->data_dir()].end()) { + _tablet_submitted_compaction[tablet->data_dir()].erase(it_tablet); + } + }); + } else { + _compaction_thread_pool->submit_func([this, tablet, permits]() { + CgroupsMgr::apply_system_cgroup(); + _perform_base_compaction(tablet); + _permit_limiter.release(permits); + std::unique_lock lock(_tablet_submitted_compaction_mutex); + vector::iterator it_tablet = find( + _tablet_submitted_compaction[tablet->data_dir()].begin(), + _tablet_submitted_compaction[tablet->data_dir()].end(), tablet); + if (it_tablet != + _tablet_submitted_compaction[tablet->data_dir()].end()) { + _tablet_submitted_compaction[tablet->data_dir()].erase(it_tablet); + } + }); + } + std::unique_lock lock(_tablet_submitted_compaction_mutex); + _tablet_submitted_compaction[tablet->data_dir()].emplace_back(tablet); } - if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - _compaction_thread_pool->submit_func([this, tablet, permits]() { - CgroupsMgr::apply_system_cgroup(); - this->_perform_cumulative_compaction(tablet); - this->_permit_limiter.release(permits); - _wait_permits_cv.notify_one(); - std::unique_lock lock(_tablet_submitted_compaction_mutex); - vector::iterator it_tablet = find( - _tablet_submitted_compaction[tablet->data_dir()].begin(), - _tablet_submitted_compaction[tablet->data_dir()].end(), tablet); - if (it_tablet != _tablet_submitted_compaction[tablet->data_dir()].end()) { - _tablet_submitted_compaction[tablet->data_dir()].erase(it_tablet); - } - }); - } else { - _compaction_thread_pool->submit_func([this, tablet, permits]() { - CgroupsMgr::apply_system_cgroup(); - this->_perform_base_compaction(tablet); - this->_permit_limiter.release(permits); - _wait_permits_cv.notify_one(); - std::unique_lock lock(_tablet_submitted_compaction_mutex); - vector::iterator it_tablet = find( - _tablet_submitted_compaction[tablet->data_dir()].begin(), - _tablet_submitted_compaction[tablet->data_dir()].end(), tablet); - if (it_tablet != _tablet_submitted_compaction[tablet->data_dir()].end()) { - _tablet_submitted_compaction[tablet->data_dir()].erase(it_tablet); - } - }); - } - std::unique_lock lock(_tablet_submitted_compaction_mutex); - _tablet_submitted_compaction[tablet->data_dir()].emplace_back(tablet); } } else { sleep(config::check_auto_compaction_interval_seconds); @@ -389,16 +386,13 @@ vector StorageEngine::_compaction_tasks_generator( vector tablets_compaction; std::random_shuffle(data_dirs.begin(), data_dirs.end()); for (auto data_dir : data_dirs) { + std::unique_lock lock(_tablet_submitted_compaction_mutex); if (_tablet_submitted_compaction[data_dir].size() >= config::compaction_task_num_per_disk) { continue; } if (!data_dir->reach_capacity_limit(0)) { - TabletSharedPtr tablet; - { - std::unique_lock lock(_tablet_submitted_compaction_mutex); - tablet = _tablet_manager->find_best_tablet_to_compaction( - compaction_type, data_dir, _tablet_submitted_compaction[data_dir]); - } + TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction( + compaction_type, data_dir, _tablet_submitted_compaction[data_dir]); if (tablet != nullptr) { tablets_compaction.emplace_back(tablet); } diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 600e90ac8e2338..ed19d68f90f002 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -350,9 +350,6 @@ class StorageEngine { std::map> _tablet_submitted_compaction; std::mutex _tablet_submitted_compaction_mutex; - std::mutex _wait_permits_mutex; - std::condition_variable _wait_permits_cv; - DISALLOW_COPY_AND_ASSIGN(StorageEngine); }; From 3e49f5339bac6b0203dda9c14a0c02adb2e6ff04 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Mon, 28 Sep 2020 11:13:18 +0800 Subject: [PATCH 25/27] modify compaction tasks producer --- be/src/common/config.h | 3 --- be/src/olap/compaction_permit_limiter.cpp | 30 ++++++++++------------- be/src/olap/compaction_permit_limiter.h | 6 ++--- be/src/olap/olap_server.cpp | 4 +-- 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index 8f10dd06416ead..fcad45fdab9bc6 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -316,9 +316,6 @@ namespace config { // The upper limit of "permits" held by all compaction tasks. This config can be set to limit memory consumption for compaction. CONF_mInt64(total_permits_for_compaction_score, "10000"); - // Whether compaction task is allowed to start when compaction score of current tablet is out of upper limit. - CONF_mBool(enable_compaction_permit_over_sold, "true"); - // Compaction task number per disk. CONF_mInt32(compaction_task_num_per_disk, "2"); diff --git a/be/src/olap/compaction_permit_limiter.cpp b/be/src/olap/compaction_permit_limiter.cpp index 39f3add4e56992..b0a08e6679db94 100644 --- a/be/src/olap/compaction_permit_limiter.cpp +++ b/be/src/olap/compaction_permit_limiter.cpp @@ -23,32 +23,28 @@ CompactionPermitLimiter::CompactionPermitLimiter() : _used_permits(0) {} bool CompactionPermitLimiter::request(int64_t permits) { if (permits > config::total_permits_for_compaction_score) { - if (config::enable_compaction_permit_over_sold) { - std::unique_lock lock(_over_sold_mutex); - _over_sold_cv.wait(lock, [=] { - return _used_permits == 0 || - _used_permits + permits <= config::total_permits_for_compaction_score; - }); - _used_permits += permits; - return true; - } else { - return false; - } + // when tablet's compaction score is larger than "config::total_permits_for_compaction_score", + // it's necessary to do compaction for this tablet because this tablet will not get "permits" + // anyway. otherwise, compaction task for this tablet will not be executed forever. + std::unique_lock lock(_permits_mutex); + _permits_cv.wait(lock, [=] { + return _used_permits == 0 || + _used_permits + permits <= config::total_permits_for_compaction_score; + }); } else { if (_used_permits + permits > config::total_permits_for_compaction_score) { - std::unique_lock lock(_wait_permits_mutex); - _wait_permits_cv.wait(lock, [=] { + std::unique_lock lock(_permits_mutex); + _permits_cv.wait(lock, [=] { return _used_permits + permits <= config::total_permits_for_compaction_score; }); } - _used_permits += permits; - return true; } + _used_permits += permits; + return true; } void CompactionPermitLimiter::release(int64_t permits) { _used_permits -= permits; - _over_sold_cv.notify_one(); - _wait_permits_cv.notify_one(); + _permits_cv.notify_one(); } } // namespace doris diff --git a/be/src/olap/compaction_permit_limiter.h b/be/src/olap/compaction_permit_limiter.h index 0c4c4e2d0a4369..b8a216d361ff33 100644 --- a/be/src/olap/compaction_permit_limiter.h +++ b/be/src/olap/compaction_permit_limiter.h @@ -43,9 +43,7 @@ class CompactionPermitLimiter { private: // sum of "permits" held by executing compaction tasks currently AtomicInt64 _used_permits; - std::mutex _over_sold_mutex; - std::condition_variable _over_sold_cv; - std::mutex _wait_permits_mutex; - std::condition_variable _wait_permits_cv; + std::mutex _permits_mutex; + std::condition_variable _permits_cv; }; } // namespace doris diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 903b9f002df96b..a516f64be16548 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -343,7 +343,7 @@ void StorageEngine::_compaction_tasks_producer_callback() { int64_t permits = tablet->calc_compaction_score(compaction_type); if (_permit_limiter.request(permits)) { if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { - _compaction_thread_pool->submit_func([this, tablet, permits]() { + _compaction_thread_pool->submit_func([=]() { CgroupsMgr::apply_system_cgroup(); _perform_cumulative_compaction(tablet); _permit_limiter.release(permits); @@ -357,7 +357,7 @@ void StorageEngine::_compaction_tasks_producer_callback() { } }); } else { - _compaction_thread_pool->submit_func([this, tablet, permits]() { + _compaction_thread_pool->submit_func([=]() { CgroupsMgr::apply_system_cgroup(); _perform_base_compaction(tablet); _permit_limiter.release(permits); From ab5f6b30aa44e39c5b91104b4b63403ffd278637 Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Mon, 28 Sep 2020 19:21:36 +0800 Subject: [PATCH 26/27] optimize compaction memory and code refactor --- be/src/olap/olap_server.cpp | 32 ++++++++++++++++++++++++-------- be/src/olap/storage_engine.h | 8 +++++++- be/src/olap/tablet_manager.cpp | 6 +++--- be/src/olap/tablet_manager.h | 2 +- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index a516f64be16548..57ed54d58a75c0 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -319,7 +319,7 @@ void StorageEngine::_compaction_tasks_producer_callback() { #endif LOG(INFO) << "try to start compaction producer process!"; - std::vector tablet_submitted; + std::vector tablet_submitted; std::vector data_dirs; for (auto& tmp_store : _store_map) { data_dirs.push_back(tmp_store.second); @@ -339,6 +339,15 @@ void StorageEngine::_compaction_tasks_producer_callback() { } vector tablets_compaction = _compaction_tasks_generator(compaction_type, data_dirs); + if (tablets_compaction.size() == 0) { + _wakeup_producer_flag = 0; + std::unique_lock lock(_compaction_producer_sleep_mutex); + // It is necessary to wake up the thread on timeout to prevent deadlock + // in case of no running compaction task. + _compaction_producer_sleep_cv.wait_for(lock, std::chrono::milliseconds(2000), + [=] { return _wakeup_producer_flag == 1; }); + continue; + } for (const auto& tablet : tablets_compaction) { int64_t permits = tablet->calc_compaction_score(compaction_type); if (_permit_limiter.request(permits)) { @@ -348,12 +357,15 @@ void StorageEngine::_compaction_tasks_producer_callback() { _perform_cumulative_compaction(tablet); _permit_limiter.release(permits); std::unique_lock lock(_tablet_submitted_compaction_mutex); - vector::iterator it_tablet = find( - _tablet_submitted_compaction[tablet->data_dir()].begin(), - _tablet_submitted_compaction[tablet->data_dir()].end(), tablet); + vector::iterator it_tablet = + find(_tablet_submitted_compaction[tablet->data_dir()].begin(), + _tablet_submitted_compaction[tablet->data_dir()].end(), + tablet->tablet_id()); if (it_tablet != _tablet_submitted_compaction[tablet->data_dir()].end()) { _tablet_submitted_compaction[tablet->data_dir()].erase(it_tablet); + _wakeup_producer_flag = 1; + _compaction_producer_sleep_cv.notify_one(); } }); } else { @@ -362,17 +374,21 @@ void StorageEngine::_compaction_tasks_producer_callback() { _perform_base_compaction(tablet); _permit_limiter.release(permits); std::unique_lock lock(_tablet_submitted_compaction_mutex); - vector::iterator it_tablet = find( - _tablet_submitted_compaction[tablet->data_dir()].begin(), - _tablet_submitted_compaction[tablet->data_dir()].end(), tablet); + vector::iterator it_tablet = + find(_tablet_submitted_compaction[tablet->data_dir()].begin(), + _tablet_submitted_compaction[tablet->data_dir()].end(), + tablet->tablet_id()); if (it_tablet != _tablet_submitted_compaction[tablet->data_dir()].end()) { _tablet_submitted_compaction[tablet->data_dir()].erase(it_tablet); + _wakeup_producer_flag = 1; + _compaction_producer_sleep_cv.notify_one(); } }); } std::unique_lock lock(_tablet_submitted_compaction_mutex); - _tablet_submitted_compaction[tablet->data_dir()].emplace_back(tablet); + _tablet_submitted_compaction[tablet->data_dir()].emplace_back( + tablet->tablet_id()); } } } else { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index ed19d68f90f002..f5707239d7649e 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -347,8 +347,14 @@ class StorageEngine { std::unique_ptr _compaction_thread_pool; CompactionPermitLimiter _permit_limiter; - std::map> _tablet_submitted_compaction; + std::mutex _tablet_submitted_compaction_mutex; + std::map> _tablet_submitted_compaction; + + AtomicInt32 _wakeup_producer_flag; + + std::mutex _compaction_producer_sleep_mutex; + std::condition_variable _compaction_producer_sleep_cv; DISALLOW_COPY_AND_ASSIGN(StorageEngine); }; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index c034516bc7558d..fdfa99631c80a6 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -681,7 +681,7 @@ void TabletManager::get_tablet_stat(TTabletStatResult* result) { TabletSharedPtr TabletManager::find_best_tablet_to_compaction( CompactionType compaction_type, DataDir* data_dir, - vector& tablet_submitted_compaction) { + vector &tablet_submitted_compaction) { int64_t now_ms = UnixMillis(); const string& compaction_type_str = compaction_type == CompactionType::BASE_COMPACTION ? "base" : "cumulative"; uint32_t highest_score = 0; @@ -691,9 +691,9 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction( tablet_map_t& tablet_map = _tablet_map_array[i]; for (tablet_map_t::value_type& table_ins : tablet_map){ for (TabletSharedPtr& tablet_ptr : table_ins.second.table_arr) { - vector::iterator it_tablet = + vector::iterator it_tablet = find(tablet_submitted_compaction.begin(), tablet_submitted_compaction.end(), - tablet_ptr); + tablet_ptr->tablet_id()); if (it_tablet != tablet_submitted_compaction.end()) { continue; } diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index 6f9c0f80c295cd..dda2f9ef207990 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -71,7 +71,7 @@ class TabletManager { TabletSharedPtr find_best_tablet_to_compaction( CompactionType compaction_type, DataDir* data_dir, - vector& tablet_submitted_compaction); + vector &tablet_submitted_compaction); TabletSharedPtr get_tablet(TTabletId tablet_id, SchemaHash schema_hash, bool include_deleted = false, std::string* err = nullptr); From a0206b3586aa17d86a2e12e445fcf94b5f32c74c Mon Sep 17 00:00:00 2001 From: weizuo93 Date: Wed, 30 Sep 2020 10:26:11 +0800 Subject: [PATCH 27/27] compaction log modification --- be/src/olap/compaction.cpp | 7 ++++--- be/src/olap/compaction.h | 2 +- be/src/olap/storage_engine.cpp | 2 +- be/src/olap/tablet_manager.cpp | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index f81a029cc1a56b..aa0f0b4523b94f 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -39,13 +39,13 @@ OLAPStatus Compaction::do_compaction(int64_t permits) { TRACE("start to do compaction"); _tablet->data_dir()->disks_compaction_score_increment(permits); _tablet->data_dir()->disks_compaction_num_increment(1); - OLAPStatus st = do_compaction_impl(); + OLAPStatus st = do_compaction_impl(permits); _tablet->data_dir()->disks_compaction_score_increment(-permits); _tablet->data_dir()->disks_compaction_num_increment(-1); return st; } -OLAPStatus Compaction::do_compaction_impl() { +OLAPStatus Compaction::do_compaction_impl(int64_t permits) { OlapStopWatch watch; // 1. prepare input and output parameters @@ -63,7 +63,8 @@ OLAPStatus Compaction::do_compaction_impl() { _tablet->compute_version_hash_from_rowsets(_input_rowsets, &_output_version_hash); LOG(INFO) << "start " << compaction_name() << ". tablet=" << _tablet->full_name() - << ", output version is=" << _output_version.first << "-" << _output_version.second; + << ", output version is=" << _output_version.first << "-" << _output_version.second + << ", score: " << permits; RETURN_NOT_OK(construct_output_rowset_writer()); RETURN_NOT_OK(construct_input_rowset_readers()); diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h index 20a567cdc82bed..f43bc6f1dcc2a4 100644 --- a/be/src/olap/compaction.h +++ b/be/src/olap/compaction.h @@ -55,7 +55,7 @@ class Compaction { virtual ReaderType compaction_type() const = 0; OLAPStatus do_compaction(int64_t permits); - OLAPStatus do_compaction_impl(); + OLAPStatus do_compaction_impl(int64_t permits); void modify_rowsets(); OLAPStatus gc_unused_rowsets(); diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 020b40fca3ec8f..c2cfcc1627fa6e 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -598,8 +598,8 @@ void StorageEngine::_perform_cumulative_compaction(TabletSharedPtr best_tablet) OLAPStatus res = cumulative_compaction.compact(); if (res != OLAP_SUCCESS) { - best_tablet->set_last_cumu_compaction_failure_time(UnixMillis()); if (res != OLAP_ERR_CUMULATIVE_NO_SUITABLE_VERSIONS) { + best_tablet->set_last_cumu_compaction_failure_time(UnixMillis()); DorisMetrics::instance()->cumulative_compaction_request_failed->increment(1); LOG(WARNING) << "failed to do cumulative compaction. res=" << res << ", table=" << best_tablet->full_name(); diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index fdfa99631c80a6..bf7d09890dac89 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -763,7 +763,7 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction( } if (best_tablet != nullptr) { - LOG(INFO) << "Found the best tablet for compaction. " + VLOG(1) << "Found the best tablet for compaction. " << "compaction_type=" << compaction_type_str << ", tablet_id=" << best_tablet->tablet_id() << ", highest_score=" << highest_score;