From 5bb7b92a54bb4d80193489a043d53fc30a078cf6 Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Tue, 31 Mar 2020 14:51:17 +0800 Subject: [PATCH 1/5] [config] Support to modify configs when BE is running without restarting it In the past, when we want to modify some BE configs, we have to modify be.conf and then restart BE. This patch provides a way to modify configs in the type of 'threshold', 'interval', 'enable flag' when BE is running without restarting it. You can update a single config once by BE's http API: `be_host:be_http_port/api/update_config?config_name=new_value` --- be/src/common/config.h | 172 ++++++------ be/src/common/configbase.cpp | 259 +++++++++++-------- be/src/common/configbase.h | 101 ++++---- be/src/common/logging.h | 1 - be/src/http/CMakeLists.txt | 1 + be/src/http/action/update_config_action.cpp | 76 ++++++ be/src/http/action/update_config_action.h | 33 +++ be/src/http/default_path_handlers.cpp | 2 +- be/src/http/http_client.cpp | 1 + be/src/olap/olap_meta.cpp | 11 +- be/src/olap/olap_server.cpp | 96 ++++--- be/src/plugin/plugin_mgr.cpp | 3 +- be/src/runtime/external_scan_context_mgr.cpp | 4 +- be/src/runtime/external_scan_context_mgr.h | 1 - be/src/runtime/load_channel_mgr.cpp | 2 +- be/src/runtime/mem_pool.h | 1 + be/src/runtime/thread_resource_mgr.cpp | 1 + be/src/service/backend_options.cpp | 1 + be/src/service/http_service.cpp | 4 + be/src/util/metrics.h | 1 + be/src/util/runtime_profile.cpp | 1 + be/src/util/thrift_rpc_helper.h | 1 + be/test/common/config_test.cpp | 75 +++++- be/test/common/resource_tls_test.cpp | 1 + be/test/exec/tablet_sink_test.cpp | 2 - be/test/exprs/hybird_set_test.cpp | 1 + be/test/olap/bloom_filter_test.cpp | 1 + be/test/olap/byte_buffer_test.cpp | 1 + be/test/olap/file_helper_test.cpp | 1 + be/test/olap/file_utils_test.cpp | 1 + be/test/runtime/large_int_value_test.cpp | 1 + be/test/util/cidr_test.cpp | 1 + be/test/util/filesystem_util_test.cpp | 1 + be/test/util/internal_queue_test.cpp | 1 + 34 files changed, 560 insertions(+), 300 deletions(-) create mode 100644 be/src/http/action/update_config_action.cpp create mode 100644 be/src/http/action/update_config_action.h diff --git a/be/src/common/config.h b/be/src/common/config.h index ad7513d60f2a21..9bbe0a9c80373c 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -40,9 +40,9 @@ namespace config { //// tcmalloc gc parameter //// // min memory for TCmalloc, when used memory is smaller than this, do not returned to OS - CONF_Int64(tc_use_memory_min, "10737418240"); + CONF_mInt64(tc_use_memory_min, "10737418240"); // free memory rate.[0-100] - CONF_Int64(tc_free_memory_rate, "20"); + CONF_mInt64(tc_free_memory_rate, "20"); // process memory limit specified as number of bytes // ('[bB]?'), megabytes ('[mM]'), gigabytes ('[gG]'), @@ -87,29 +87,29 @@ namespace config { // the count of thread to release snapshot CONF_Int32(release_snapshot_worker_count, "5"); // the interval time(seconds) for agent report tasks signatrue to FE - CONF_Int32(report_task_interval_seconds, "10"); + CONF_mInt32(report_task_interval_seconds, "10"); // the interval time(seconds) for agent report disk state to FE - CONF_Int32(report_disk_state_interval_seconds, "60"); + CONF_mInt32(report_disk_state_interval_seconds, "60"); // the interval time(seconds) for agent report olap table to FE - CONF_Int32(report_tablet_interval_seconds, "60"); + CONF_mInt32(report_tablet_interval_seconds, "60"); // the interval time(seconds) for agent report plugin status to FE // CONF_Int32(report_plugin_interval_seconds, "120"); // the timeout(seconds) for alter table - CONF_Int32(alter_tablet_timeout_seconds, "86400"); + // CONF_Int32(alter_tablet_timeout_seconds, "86400"); // the timeout(seconds) for make snapshot - CONF_Int32(make_snapshot_timeout_seconds, "600"); + // CONF_Int32(make_snapshot_timeout_seconds, "600"); // the timeout(seconds) for release snapshot - CONF_Int32(release_snapshot_timeout_seconds, "600"); + // CONF_Int32(release_snapshot_timeout_seconds, "600"); // the max download speed(KB/s) - CONF_Int32(max_download_speed_kbps, "50000"); + CONF_mInt32(max_download_speed_kbps, "50000"); // download low speed limit(KB/s) - CONF_Int32(download_low_speed_limit_kbps, "50"); + CONF_mInt32(download_low_speed_limit_kbps, "50"); // download low speed time(seconds) - CONF_Int32(download_low_speed_time, "300"); + CONF_mInt32(download_low_speed_time, "300"); // curl verbose mode - CONF_Int64(curl_verbose_mode, "1"); + // CONF_Int64(curl_verbose_mode, "1"); // seconds to sleep for each time check table status - CONF_Int32(check_status_sleep_time_seconds, "10"); + // CONF_Int32(check_status_sleep_time_seconds, "10"); // sleep time for one second CONF_Int32(sleep_one_second, "1"); // sleep time for five seconds @@ -142,7 +142,7 @@ namespace config { CONF_String(default_query_options, ""); // If non-zero, Doris will output memory usage every log_mem_usage_interval'th fragment completion. - CONF_Int32(log_mem_usage_interval, "0"); + // CONF_Int32(log_mem_usage_interval, "0"); // cgroups allocated for doris CONF_String(doris_cgroups, ""); @@ -156,7 +156,7 @@ namespace config { // serialize and deserialize each returned row batch CONF_Bool(serialize_batch, "false"); // interval between profile reports; in seconds - CONF_Int32(status_report_interval, "5"); + CONF_mInt32(status_report_interval, "5"); // Local directory to copy UDF libraries from HDFS into CONF_String(local_library_dir, "${UDF_RUNTIME_DIR}"); // number of olap scanner thread pool size @@ -172,63 +172,63 @@ namespace config { // default thrift client connect timeout(in seconds) CONF_Int32(thrift_connect_timeout_seconds, "3"); // max row count number for single scan range - CONF_Int32(doris_scan_range_row_count, "524288"); + CONF_mInt32(doris_scan_range_row_count, "524288"); // size of scanner queue between scanner thread and compute thread - CONF_Int32(doris_scanner_queue_size, "1024"); + CONF_mInt32(doris_scanner_queue_size, "1024"); // single read execute fragment row size - CONF_Int32(doris_scanner_row_num, "16384"); + CONF_mInt32(doris_scanner_row_num, "16384"); // number of max scan keys - CONF_Int32(doris_max_scan_key_num, "1024"); + CONF_mInt32(doris_max_scan_key_num, "1024"); // return_row / total_row - CONF_Int32(doris_max_pushdown_conjuncts_return_rate, "90"); + CONF_mInt32(doris_max_pushdown_conjuncts_return_rate, "90"); // (Advanced) Maximum size of per-query receive-side buffer - CONF_Int32(exchg_node_buffer_size_bytes, "10485760"); + CONF_mInt32(exchg_node_buffer_size_bytes, "10485760"); // insert sort threadhold for sorter - CONF_Int32(insertion_threadhold, "16"); + // CONF_Int32(insertion_threadhold, "16"); // the block_size every block allocate for sorter CONF_Int32(sorter_block_size, "8388608"); // push_write_mbytes_per_sec CONF_Int32(push_write_mbytes_per_sec, "10"); - CONF_Int64(column_dictionary_key_ratio_threshold, "0"); - CONF_Int64(column_dictionary_key_size_threshold, "0"); + CONF_mInt64(column_dictionary_key_ratio_threshold, "0"); + CONF_mInt64(column_dictionary_key_size_threshold, "0"); // if true, output IR after optimization passes - CONF_Bool(dump_ir, "false"); + // CONF_Bool(dump_ir, "false"); // if set, saves the generated IR to the output file. - CONF_String(module_output, ""); + //CONF_String(module_output, ""); // memory_limitation_per_thread_for_schema_change unit GB - CONF_Int32(memory_limitation_per_thread_for_schema_change, "2"); + CONF_mInt32(memory_limitation_per_thread_for_schema_change, "2"); - CONF_Int64(max_unpacked_row_block_size, "104857600"); + // CONF_Int64(max_unpacked_row_block_size, "104857600"); - CONF_Int32(file_descriptor_cache_clean_interval, "3600"); - CONF_Int32(disk_stat_monitor_interval, "5"); - CONF_Int32(unused_rowset_monitor_interval, "30"); + CONF_mInt32(file_descriptor_cache_clean_interval, "3600"); + CONF_mInt32(disk_stat_monitor_interval, "5"); + CONF_mInt32(unused_rowset_monitor_interval, "30"); CONF_String(storage_root_path, "${DORIS_HOME}/storage"); // BE process will exit if the percentage of error disk reach this value. - CONF_Int32(max_percentage_of_error_disk, "0"); - CONF_Int32(default_num_rows_per_data_block, "1024"); - CONF_Int32(default_num_rows_per_column_file_block, "1024"); + CONF_mInt32(max_percentage_of_error_disk, "0"); + // CONF_Int32(default_num_rows_per_data_block, "1024"); + CONF_mInt32(default_num_rows_per_column_file_block, "1024"); CONF_Int32(max_tablet_num_per_shard, "1024"); // pending data policy - CONF_Int32(pending_data_expire_time_sec, "1800"); + CONF_mInt32(pending_data_expire_time_sec, "1800"); // inc_rowset expired interval - CONF_Int32(inc_rowset_expired_sec, "1800"); + CONF_mInt32(inc_rowset_expired_sec, "1800"); // garbage sweep policy CONF_Int32(max_garbage_sweep_interval, "3600"); CONF_Int32(min_garbage_sweep_interval, "180"); - CONF_Int32(snapshot_expire_time_sec, "172800"); + CONF_mInt32(snapshot_expire_time_sec, "172800"); // 仅仅是建议值，当磁盘空间不足时，trash下的文件保存期可不遵守这个参数 - CONF_Int32(trash_file_expire_time_sec, "259200"); + CONF_mInt32(trash_file_expire_time_sec, "259200"); // check row nums for BE/CE and schema change. true is open, false is closed. - CONF_Bool(row_nums_check, "true") + CONF_mBool(row_nums_check, "true") //file descriptors cache, by default, cache 32768 descriptors CONF_Int32(file_descriptor_cache_capacity, "32768"); // minimum file descriptor number // modify them upon necessity CONF_Int32(min_file_descriptor_number, "60000"); CONF_Int64(index_stream_cache_capacity, "10737418240"); - CONF_Int64(max_packed_row_block_size, "20971520"); + // CONF_Int64(max_packed_row_block_size, "20971520"); // Cache for stoage page size CONF_String(storage_page_cache_limit, "20G"); @@ -236,26 +236,26 @@ namespace config { CONF_Bool(disable_storage_page_cache, "false"); // be policy - CONF_Int64(base_compaction_start_hour, "20"); - CONF_Int64(base_compaction_end_hour, "7"); - CONF_Int32(base_compaction_check_interval_seconds, "60"); - CONF_Int64(base_compaction_num_cumulative_deltas, "5"); + // CONF_Int64(base_compaction_start_hour, "20"); + // CONF_Int64(base_compaction_end_hour, "7"); + CONF_mInt32(base_compaction_check_interval_seconds, "60"); + CONF_mInt64(base_compaction_num_cumulative_deltas, "5"); CONF_Int32(base_compaction_num_threads_per_disk, "1"); - CONF_Double(base_cumulative_delta_ratio, "0.3"); - CONF_Int64(base_compaction_interval_seconds_since_last_operation, "86400"); - CONF_Int32(base_compaction_write_mbytes_per_sec, "5"); + CONF_mDouble(base_cumulative_delta_ratio, "0.3"); + CONF_mInt64(base_compaction_interval_seconds_since_last_operation, "86400"); + CONF_mInt32(base_compaction_write_mbytes_per_sec, "5"); // cumulative compaction policy: max delta file's size unit:B - CONF_Int32(cumulative_compaction_check_interval_seconds, "10"); - CONF_Int64(min_cumulative_compaction_num_singleton_deltas, "5"); - CONF_Int64(max_cumulative_compaction_num_singleton_deltas, "1000"); + CONF_mInt32(cumulative_compaction_check_interval_seconds, "10"); + CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5"); + CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000"); CONF_Int32(cumulative_compaction_num_threads_per_disk, "1"); - CONF_Int64(cumulative_compaction_budgeted_bytes, "104857600"); - CONF_Int32(cumulative_compaction_write_mbytes_per_sec, "100"); + CONF_mInt64(cumulative_compaction_budgeted_bytes, "104857600"); + // CONF_Int32(cumulative_compaction_write_mbytes_per_sec, "100"); // if compaction of a tablet failed, this tablet should not be chosen to // compaction until this interval passes. - CONF_Int64(min_compaction_failure_interval_sec, "600") // 10 min + CONF_mInt64(min_compaction_failure_interval_sec, "600") // 10 min // Too many compaction tasks may run out of memory. // This config is to limit the max concurrency of running compaction tasks. // -1 means no limit, and the max concurrency will be: @@ -269,17 +269,17 @@ namespace config { // Number of webserver workers CONF_Int32(webserver_num_workers, "5"); // Period to update rate counters and sampling counters in ms. - CONF_Int32(periodic_counter_update_period_ms, "500"); + CONF_mInt32(periodic_counter_update_period_ms, "500"); // Used for mini Load. mini load data file will be removed after this time. CONF_Int64(load_data_reserve_hours, "4"); // log error log will be removed after this time - CONF_Int64(load_error_log_reserve_hours, "48"); + CONF_mInt64(load_error_log_reserve_hours, "48"); // Deprecated, use streaming_load_max_mb instead - CONF_Int64(mini_load_max_mb, "2048"); + // CONF_Int64(mini_load_max_mb, "2048"); CONF_Int32(number_tablet_writer_threads, "16"); - CONF_Int64(streaming_load_max_mb, "10240"); + CONF_mInt64(streaming_load_max_mb, "10240"); // the alive time of a TabletsChannel. // If the channel does not receive any data till this time, // the channel will be removed. @@ -291,14 +291,14 @@ namespace config { // the timeout of a rpc to process one batch in tablet writer. // you may need to increase this timeout if using larger 'streaming_load_max_mb', // or encounter 'tablet writer write failed' error when loading. - CONF_Int32(tablet_writer_rpc_timeout_sec, "600"); + // CONF_Int32(tablet_writer_rpc_timeout_sec, "600"); // Fragment thread pool CONF_Int32(fragment_pool_thread_num, "64"); CONF_Int32(fragment_pool_queue_size, "1024"); //for cast - CONF_Bool(cast, "true"); + // CONF_Bool(cast, "true"); // Spill to disk when query // Writable scratch directories, splitted by ";" @@ -342,26 +342,26 @@ namespace config { CONF_String(pprof_profile_dir, "${DORIS_HOME}/log") // for partition - CONF_Bool(enable_partitioned_hash_join, "false") + // CONF_Bool(enable_partitioned_hash_join, "false") CONF_Bool(enable_partitioned_aggregation, "true") // to forward compatibility, will be removed later - CONF_Bool(enable_token_check, "true"); + CONF_mBool(enable_token_check, "true"); // to open/close system metrics CONF_Bool(enable_system_metrics, "true"); - CONF_Bool(enable_prefetch, "true"); + CONF_mBool(enable_prefetch, "true"); // Number of cores Doris will used, this will effect only when it's greater than 0. // Otherwise, Doris will use all cores returned from "/proc/cpuinfo". CONF_Int32(num_cores, "0"); - CONF_Bool(thread_creation_fault_injection, "false"); + // CONF_Bool(thread_creation_fault_injection, "false"); // Set this to encrypt and perform an integrity // check on all data spilled to disk during a query - CONF_Bool(disk_spill_encryption, "false"); + // CONF_Bool(disk_spill_encryption, "false"); // When BE start, If there is a broken disk, BE process will exit by default. // Otherwise, we will ignore the broken disk, @@ -372,7 +372,7 @@ namespace config { // If false and --scratch_dirs contains multiple directories on the same device, // then only the first writable directory is used - CONF_Bool(allow_multiple_scratch_dirs_per_device, "false"); + // CONF_Bool(allow_multiple_scratch_dirs_per_device, "false"); // linux transparent huge page CONF_Bool(madvise_huge_pages, "false"); @@ -387,13 +387,13 @@ namespace config { CONF_String(buffer_pool_clean_pages_limit, "20G"); // Sleep time in seconds between memory maintenance iterations - CONF_Int64(memory_maintenance_sleep_time_s, "10"); + CONF_mInt64(memory_maintenance_sleep_time_s, "10"); // Aligement CONF_Int32(memory_max_alignment, "16"); // write buffer size before flush - CONF_Int64(write_buffer_size, "104857600"); + CONF_mInt64(write_buffer_size, "104857600"); // following 2 configs limit the memory consumption of load process on a Backend. // eg: memory limit to 80% of mem limit config but up to 100GB(default) @@ -404,31 +404,31 @@ namespace config { CONF_Int32(load_process_max_memory_limit_percent, "80"); // 80% // update interval of tablet stat cache - CONF_Int32(tablet_stat_cache_update_interval_second, "300"); + CONF_mInt32(tablet_stat_cache_update_interval_second, "300"); // result buffer cancelled time (unit: second) - CONF_Int32(result_buffer_cancelled_interval_time, "300"); + CONF_mInt32(result_buffer_cancelled_interval_time, "300"); // can perform recovering tablet CONF_Bool(force_recovery, "false"); // the increased frequency of priority for remaining tasks in BlockingPriorityQueue - CONF_Int32(priority_queue_remaining_tasks_increased_frequency, "512"); + CONF_mInt32(priority_queue_remaining_tasks_increased_frequency, "512"); // sync tablet_meta when modifing meta - CONF_Bool(sync_tablet_meta, "false"); + CONF_mBool(sync_tablet_meta, "false"); // default thrift rpc timeout ms - CONF_Int32(thrift_rpc_timeout_ms, "5000"); + CONF_mInt32(thrift_rpc_timeout_ms, "5000"); // txn commit rpc timeout - CONF_Int32(txn_commit_rpc_timeout_ms, "10000"); + CONF_mInt32(txn_commit_rpc_timeout_ms, "10000"); // If set to true, metric calculator will run CONF_Bool(enable_metric_calculator, "true"); // max consumer num in one data consumer group, for routine load - CONF_Int32(max_consumer_num_per_group, "3"); + CONF_mInt32(max_consumer_num_per_group, "3"); // the size of thread pool for routine load task. // this should be larger than FE config 'max_concurrent_task_num_per_be' (default 5) @@ -436,15 +436,15 @@ namespace config { // Is set to true, index loading failure will not causing BE exit, // and the tablet will be marked as bad, so that FE will try to repair it. - CONF_Bool(auto_recover_index_loading_failure, "false"); + // CONF_Bool(auto_recover_index_loading_failure, "false"); // max external scan cache batch count, means cache max_memory_cache_batch_count * batch_size row // default is 20, batch_size's defualt value is 1024 means 20 * 1024 rows will be cached - CONF_Int32(max_memory_sink_batch_count, "20"); + CONF_mInt32(max_memory_sink_batch_count, "20"); // This configuration is used for the context gc thread schedule period // note: unit is minute, default is 5min - CONF_Int32(scan_context_gc_interval_min, "5"); + CONF_mInt32(scan_context_gc_interval_min, "5"); // es scroll keep-alive CONF_String(es_scroll_keepalive, "5m"); @@ -463,22 +463,22 @@ namespace config { // path gc CONF_Bool(path_gc_check, "true"); CONF_Int32(path_gc_check_interval_second, "86400"); - CONF_Int32(path_gc_check_step, "1000"); - CONF_Int32(path_gc_check_step_interval_ms, "10"); - CONF_Int32(path_scan_interval_second, "86400"); + CONF_mInt32(path_gc_check_step, "1000"); + CONF_mInt32(path_gc_check_step_interval_ms, "10"); + CONF_mInt32(path_scan_interval_second, "86400"); // The following 2 configs limit the max usage of disk capacity of a data dir. // If both of these 2 threshold reached, no more data can be writen into that data dir. // The percent of max used capacity of a data dir - CONF_Int32(storage_flood_stage_usage_percent, "95"); // 95% + CONF_mInt32(storage_flood_stage_usage_percent, "95"); // 95% // The min bytes that should be left of a data dir - CONF_Int64(storage_flood_stage_left_capacity_bytes, "1073741824") // 1GB + CONF_mInt64(storage_flood_stage_left_capacity_bytes, "1073741824") // 1GB // number of thread for flushing memtable per store CONF_Int32(flush_thread_num_per_store, "2"); // config for tablet meta checkpoint - CONF_Int32(tablet_meta_checkpoint_min_new_rowsets_num, "10"); - CONF_Int32(tablet_meta_checkpoint_min_interval_secs, "600"); + CONF_mInt32(tablet_meta_checkpoint_min_new_rowsets_num, "10"); + CONF_mInt32(tablet_meta_checkpoint_min_interval_secs, "600"); // config for default rowset type // Valid configs: ALPHA, BETA @@ -489,12 +489,12 @@ namespace config { // max number of txns in txn manager // this is a self protection to avoid too many txns saving in manager - CONF_Int64(max_runnings_transactions, "2000"); + CONF_mInt64(max_runnings_transactions, "2000"); // tablet_map_lock shard size, the value is 2^n, n=0,1,2,3,4 // this is a an enhancement for better performance to manage tablet CONF_Int32(tablet_map_shard_size, "1"); - + CONF_String(plugin_path, "${DORIS_HOME}/plugin") } // namespace config diff --git a/be/src/common/configbase.cpp b/be/src/common/configbase.cpp index 9f3673b8a4c322..9a8d8b633f2081 100644 --- a/be/src/common/configbase.cpp +++ b/be/src/common/configbase.cpp @@ -28,91 +28,19 @@ #include "common/config.h" #undef __IN_CONFIGBASE_CPP__ +#include "common/status.h" +#include "gutil/strings/substitute.h" + namespace doris { namespace config { -std::list* Register::_s_fieldlist = nullptr; -std::map* confmap = nullptr; +std::map* Register::_s_fieldlist = nullptr; +std::map* full_conf_map = nullptr; Properties props; -// load conf file -bool Properties::load(const char* filename) { - // if filename is null, use the empty props - if (filename == nullptr) { - return true; - } - - // open the conf file - std::ifstream input(filename); - if (!input.is_open()) { - std::cerr << "config::load() failed to open the file:" << filename << std::endl; - return false; - } - - // load properties - std::string line; - std::string key; - std::string value; - line.reserve(512); - while (input) { - // read one line at a time - std::getline(input, line); - - // remove left and right spaces - trim(line); - - // ignore comments - if (line.empty() || line[0] == '#') { - continue; - } - - // read key and value - splitkv(line, key, value); - trim(key); - trim(value); - - // insert into propmap - propmap[key] = value; - } - - // close the conf file - input.close(); - - return true; -} - -template -bool Properties::get(const char* key, const char* defstr, T& retval) const { - std::map::const_iterator it = propmap.find(std::string(key)); - std::string valstr = it != propmap.end() ? it->second : std::string(defstr); - trim(valstr); - if (!replaceenv(valstr)) { - return false; - } - return strtox(valstr, retval); -} - -template -bool Properties::strtox(const std::string& valstr, std::vector& retval) { - std::stringstream ss(valstr); - std::string item; - T t; - while (std::getline(ss, item, ',')) { - if (!strtox(trim(item), t)) { - return false; - } - retval.push_back(t); - } - return true; -} - -const std::map& Properties::getmap() const { - return propmap; -} - // trim string -std::string& Properties::trim(std::string& s) { +std::string& trim(std::string& s) { // rtrim s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun(std::isspace))) .base(), @@ -124,7 +52,7 @@ std::string& Properties::trim(std::string& s) { } // split string by '=' -void Properties::splitkv(const std::string& s, std::string& k, std::string& v) { +void splitkv(const std::string& s, std::string& k, std::string& v) { const char sep = '='; int start = 0; int end = 0; @@ -138,7 +66,7 @@ void Properties::splitkv(const std::string& s, std::string& k, std::string& v) { } // replace env variables -bool Properties::replaceenv(std::string& s) { +bool replaceenv(std::string& s) { std::size_t pos = 0; std::size_t start = 0; while ((start = s.find("${", pos)) != std::string::npos) { @@ -158,7 +86,28 @@ bool Properties::replaceenv(std::string& s) { return true; } -bool Properties::strtox(const std::string& valstr, bool& retval) { +bool strtox(const std::string& valstr, bool& retval); +bool strtox(const std::string& valstr, int16_t& retval); +bool strtox(const std::string& valstr, int32_t& retval); +bool strtox(const std::string& valstr, int64_t& retval); +bool strtox(const std::string& valstr, double& retval); +bool strtox(const std::string& valstr, std::string& retval); + +template +bool strtox(const std::string& valstr, std::vector& retval) { + std::stringstream ss(valstr); + std::string item; + T t; + while (std::getline(ss, item, ',')) { + if (!strtox(trim(item), t)) { + return false; + } + retval.push_back(t); + } + return true; +} + +bool strtox(const std::string& valstr, bool& retval) { if (valstr.compare("true") == 0) { retval = true; } else if (valstr.compare("false") == 0) { @@ -170,7 +119,7 @@ bool Properties::strtox(const std::string& valstr, bool& retval) { } template -bool Properties::strtointeger(const std::string& valstr, T& retval) { +bool strtointeger(const std::string& valstr, T& retval) { if (valstr.length() == 0) { return false; // empty-string is only allowed for string type. } @@ -181,26 +130,28 @@ bool Properties::strtointeger(const std::string& valstr, T& retval) { if (errno || end != valcstr + strlen(valcstr)) { return false; // bad parse } + T tmp = retval; retval = static_cast(ret64); if (retval != ret64) { + retval = tmp; return false; } return true; } -bool Properties::strtox(const std::string& valstr, int16_t& retval) { +bool strtox(const std::string& valstr, int16_t& retval) { return strtointeger(valstr, retval); } -bool Properties::strtox(const std::string& valstr, int32_t& retval) { +bool strtox(const std::string& valstr, int32_t& retval) { return strtointeger(valstr, retval); } -bool Properties::strtox(const std::string& valstr, int64_t& retval) { +bool strtox(const std::string& valstr, int64_t& retval) { return strtointeger(valstr, retval); } -bool Properties::strtox(const std::string& valstr, double& retval) { +bool strtox(const std::string& valstr, double& retval) { if (valstr.length() == 0) { return false; // empty-string is only allowed for string type. } @@ -214,11 +165,78 @@ bool Properties::strtox(const std::string& valstr, double& retval) { return true; } -bool Properties::strtox(const std::string& valstr, std::string& retval) { +bool strtox(const std::string& valstr, std::string& retval) { retval = valstr; return true; } +// load conf file +bool Properties::load(const char* filename) { + // if filename is null, use the empty props + if (filename == nullptr) { + return true; + } + + // open the conf file + std::ifstream input(filename); + if (!input.is_open()) { + std::cerr << "config::load() failed to open the file:" << filename << std::endl; + return false; + } + + // load properties + std::string line; + std::string key; + std::string value; + line.reserve(512); + while (input) { + // read one line at a time + std::getline(input, line); + + // remove left and right spaces + trim(line); + + // ignore comments + if (line.empty() || line[0] == '#') { + continue; + } + + // read key and value + splitkv(line, key, value); + trim(key); + trim(value); + + // insert into file_conf_map + file_conf_map[key] = value; + } + + // close the conf file + input.close(); + + return true; +} + +template +bool Properties::get(const char* key, const char* defstr, T& retval) const { + const auto& it = file_conf_map.find(std::string(key)); + std::string valstr = it != file_conf_map.end() ? it->second : std::string(defstr); + trim(valstr); + if (!replaceenv(valstr)) { + return false; + } + return strtox(valstr, retval); +} + +template +bool update(const std::string& value, T& retval) { + std::string valstr(value); + trim(valstr); + if (!replaceenv(valstr)) { + return false; + } + return strtox(valstr, retval); +} + template std::ostream& operator<<(std::ostream& out, const std::vector& v) { size_t last = v.size() - 1; @@ -240,7 +258,7 @@ std::ostream& operator<<(std::ostream& out, const std::vector& v) { if (FILL_CONFMAP) { \ std::ostringstream oss; \ oss << (*reinterpret_cast((FIELD).storage)); \ - (*confmap)[(FIELD).name] = oss.str(); \ + (*full_conf_map)[(FIELD).name] = oss.str(); \ } \ continue; \ } @@ -251,29 +269,64 @@ bool init(const char* filename, bool fillconfmap) { if (!props.load(filename)) { return false; } - // fill confmap ? - if (fillconfmap && confmap == nullptr) { - confmap = new std::map(); + // fill full_conf_map ? + if (fillconfmap && full_conf_map == nullptr) { + full_conf_map = new std::map(); } // set conf fields for (const auto& it : *Register::_s_fieldlist) { - SET_FIELD(it, bool, fillconfmap); - SET_FIELD(it, int16_t, fillconfmap); - SET_FIELD(it, int32_t, fillconfmap); - SET_FIELD(it, int64_t, fillconfmap); - SET_FIELD(it, double, fillconfmap); - SET_FIELD(it, std::string, fillconfmap); - SET_FIELD(it, std::vector, fillconfmap); - SET_FIELD(it, std::vector, fillconfmap); - SET_FIELD(it, std::vector, fillconfmap); - SET_FIELD(it, std::vector, fillconfmap); - SET_FIELD(it, std::vector, fillconfmap); - SET_FIELD(it, std::vector, fillconfmap); + SET_FIELD(it.second, bool, fillconfmap); + SET_FIELD(it.second, int16_t, fillconfmap); + SET_FIELD(it.second, int32_t, fillconfmap); + SET_FIELD(it.second, int64_t, fillconfmap); + SET_FIELD(it.second, double, fillconfmap); + SET_FIELD(it.second, std::string, fillconfmap); + SET_FIELD(it.second, std::vector, fillconfmap); + SET_FIELD(it.second, std::vector, fillconfmap); + SET_FIELD(it.second, std::vector, fillconfmap); + SET_FIELD(it.second, std::vector, fillconfmap); + SET_FIELD(it.second, std::vector, fillconfmap); + SET_FIELD(it.second, std::vector, fillconfmap); } return true; } +#define UPDATE_FIELD(FIELD, VALUE, TYPE) \ + if (strcmp((FIELD).type, #TYPE) == 0) { \ + if (!update((VALUE), *reinterpret_cast((FIELD).storage))) { \ + return Status::InvalidArgument( \ + strings::Substitute("convert '$0' as $1 failed", VALUE, #TYPE)); \ + } \ + if (full_conf_map != nullptr) { \ + std::ostringstream oss; \ + oss << (*reinterpret_cast((FIELD).storage)); \ + (*full_conf_map)[(FIELD).name] = oss.str(); \ + } \ + return Status::OK(); \ + } + +Status set_config(const std::string& field, const std::string& value) { + auto it = Register::_s_fieldlist->find(field); + if (it == Register::_s_fieldlist->end()) { + return Status::NotFound(strings::Substitute("'$0' is not found", field)); + } + + if (!it->second.valmutable) { + return Status::NotSupported(strings::Substitute("'$0' is not support to modify", field)); + } + + UPDATE_FIELD(it->second, value, bool); + UPDATE_FIELD(it->second, value, int16_t); + UPDATE_FIELD(it->second, value, int32_t); + UPDATE_FIELD(it->second, value, int64_t); + UPDATE_FIELD(it->second, value, double); + + // The other types are not thread safe to change dynamically. + return Status::NotSupported(strings::Substitute( + "'$0' is type of '$1' which is not support to modify", field, it->second.type)); +} + } // namespace config } // namespace doris diff --git a/be/src/common/configbase.h b/be/src/common/configbase.h index a4057c62217960..64f6aa759e26c9 100644 --- a/be/src/common/configbase.h +++ b/be/src/common/configbase.h @@ -20,56 +20,72 @@ #include -#include #include +#include #include namespace doris { +class Status; + namespace config { class Register { public: struct Field { - const char* type; - const char* name; - void* storage; - const char* defval; - Field(const char* ftype, const char* fname, void* fstorage, const char* fdefval) - : type(ftype), name(fname), storage(fstorage), defval(fdefval) {} + const char* type = nullptr; + const char* name = nullptr; + void* storage = nullptr; + const char* defval = nullptr; + bool valmutable = false; + Field(const char* ftype, const char* fname, void* fstorage, const char* fdefval, + bool fvalmutable) + : type(ftype), + name(fname), + storage(fstorage), + defval(fdefval), + valmutable(fvalmutable) {} }; public: - static std::list* _s_fieldlist; + static std::map* _s_fieldlist; public: - Register(const char* ftype, const char* fname, void* fstorage, const char* fdefval) { - if (_s_fieldlist == NULL) { - _s_fieldlist = new std::list(); + Register(const char* ftype, const char* fname, void* fstorage, const char* fdefval, + bool fvalmutable) { + if (_s_fieldlist == nullptr) { + _s_fieldlist = new std::map(); } - Field field(ftype, fname, fstorage, fdefval); - _s_fieldlist->push_back(field); + Field field(ftype, fname, fstorage, fdefval, fvalmutable); + _s_fieldlist->insert(std::make_pair(std::string(fname), field)); } }; -#define DEFINE_FIELD(FIELD_TYPE, FIELD_NAME, FIELD_DEFAULT) \ - FIELD_TYPE FIELD_NAME; \ - static Register reg_##FIELD_NAME(#FIELD_TYPE, #FIELD_NAME, &FIELD_NAME, FIELD_DEFAULT); +#define DEFINE_FIELD(FIELD_TYPE, FIELD_NAME, FIELD_DEFAULT, VALMUTABLE) \ + FIELD_TYPE FIELD_NAME; \ + static Register reg_##FIELD_NAME(#FIELD_TYPE, #FIELD_NAME, &FIELD_NAME, FIELD_DEFAULT, \ + VALMUTABLE); #define DECLARE_FIELD(FIELD_TYPE, FIELD_NAME) extern FIELD_TYPE FIELD_NAME; #ifdef __IN_CONFIGBASE_CPP__ -#define CONF_Bool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr) -#define CONF_Int16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr) -#define CONF_Int32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr) -#define CONF_Int64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr) -#define CONF_Double(name, defaultstr) DEFINE_FIELD(double, name, defaultstr) -#define CONF_String(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr) -#define CONF_Bools(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr) -#define CONF_Int16s(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr) -#define CONF_Int32s(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr) -#define CONF_Int64s(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr) -#define CONF_Doubles(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr) -#define CONF_Strings(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr) +#define CONF_Bool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, false) +#define CONF_Int16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, false) +#define CONF_Int32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, false) +#define CONF_Int64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, false) +#define CONF_Double(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, false) +#define CONF_String(name, defaultstr) DEFINE_FIELD(std::string, name, defaultstr, false) +#define CONF_Bools(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr, false) +#define CONF_Int16s(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr, false) +#define CONF_Int32s(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr, false) +#define CONF_Int64s(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr, false) +#define CONF_Doubles(name, defaultstr) DEFINE_FIELD(std::vector, name, defaultstr, false) +#define CONF_Strings(name, defaultstr) \ + DEFINE_FIELD(std::vector, name, defaultstr, false) +#define CONF_mBool(name, defaultstr) DEFINE_FIELD(bool, name, defaultstr, true) +#define CONF_mInt16(name, defaultstr) DEFINE_FIELD(int16_t, name, defaultstr, true) +#define CONF_mInt32(name, defaultstr) DEFINE_FIELD(int32_t, name, defaultstr, true) +#define CONF_mInt64(name, defaultstr) DEFINE_FIELD(int64_t, name, defaultstr, true) +#define CONF_mDouble(name, defaultstr) DEFINE_FIELD(double, name, defaultstr, true) #else #define CONF_Bool(name, defaultstr) DECLARE_FIELD(bool, name) #define CONF_Int16(name, defaultstr) DECLARE_FIELD(int16_t, name) @@ -83,40 +99,33 @@ class Register { #define CONF_Int64s(name, defaultstr) DECLARE_FIELD(std::vector, name) #define CONF_Doubles(name, defaultstr) DECLARE_FIELD(std::vector, name) #define CONF_Strings(name, defaultstr) DECLARE_FIELD(std::vector, name) +#define CONF_mBool(name, defaultstr) DECLARE_FIELD(bool, name) +#define CONF_mInt16(name, defaultstr) DECLARE_FIELD(int16_t, name) +#define CONF_mInt32(name, defaultstr) DECLARE_FIELD(int32_t, name) +#define CONF_mInt64(name, defaultstr) DECLARE_FIELD(int64_t, name) +#define CONF_mDouble(name, defaultstr) DECLARE_FIELD(double, name) #endif +// configuration properties load from config file. class Properties { public: bool load(const char* filename); template bool get(const char* key, const char* defstr, T& retval) const; - const std::map& getmap() const; - -private: - template - static bool strtox(const std::string& valstr, std::vector& retval); - template - static bool strtointeger(const std::string& valstr, T& retval); - static bool strtox(const std::string& valstr, bool& retval); - static bool strtox(const std::string& valstr, int16_t& retval); - static bool strtox(const std::string& valstr, int32_t& retval); - static bool strtox(const std::string& valstr, int64_t& retval); - static bool strtox(const std::string& valstr, double& retval); - static bool strtox(const std::string& valstr, std::string& retval); - static std::string& trim(std::string& s); - static void splitkv(const std::string& s, std::string& k, std::string& v); - static bool replaceenv(std::string& s); private: - std::map propmap; + std::map file_conf_map; }; extern Properties props; -extern std::map* confmap; +// full configurations. +extern std::map* full_conf_map; bool init(const char* filename, bool fillconfmap = false); +Status set_config(const std::string& field, const std::string& value); + } // namespace config } // namespace doris diff --git a/be/src/common/logging.h b/be/src/common/logging.h index b2021e6bc2685f..cad58f39a0fd6e 100644 --- a/be/src/common/logging.h +++ b/be/src/common/logging.h @@ -42,7 +42,6 @@ // This is including a glog internal file. We want this to expose the // function to get the stack trace. #include -#include #undef MutexLock #endif diff --git a/be/src/http/CMakeLists.txt b/be/src/http/CMakeLists.txt index 39f98634013d1d..0ee8d14517ff71 100644 --- a/be/src/http/CMakeLists.txt +++ b/be/src/http/CMakeLists.txt @@ -46,6 +46,7 @@ add_library(Webserver STATIC action/stream_load.cpp action/meta_action.cpp action/compaction_action.cpp + action/update_config_action.cpp # action/multi_start.cpp # action/multi_show.cpp # action/multi_commit.cpp diff --git a/be/src/http/action/update_config_action.cpp b/be/src/http/action/update_config_action.cpp new file mode 100644 index 00000000000000..37d5ed5d7456b9 --- /dev/null +++ b/be/src/http/action/update_config_action.cpp @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "http/action/update_config_action.h" + +#include +#include +#include +#include + +#include + +#include "common/configbase.h" +#include "common/logging.h" +#include "common/status.h" +#include "gutil/strings/substitute.h" +#include "http/http_channel.h" +#include "http/http_headers.h" +#include "http/http_request.h" +#include "http/http_response.h" +#include "http/http_status.h" + +namespace doris { + +const static std::string HEADER_JSON = "application/json"; + +void UpdateConfigAction::handle(HttpRequest* req) { + LOG(INFO) << req->debug_string(); + + bool success = true; + std::string msg; + if (req->params()->size() != 1) { + success = false; + msg = "Now only support to set a single config once, via 'config_name=new_value'."; + } else { + DCHECK(req->params()->size() == 1); + for (const auto& it : *req->params()) { + Status s = config::set_config(it.first, it.second); + success = s.ok(); + LOG(WARNING) << "set_config " << it.first << "=" << it.second + << (success ? " success" : " failed"); + if (!success) { + msg = strings::Substitute("set $0=$1 failed, reason: $2", it.first, it.second, + s.to_string()); + } + } + } + + std::string status(success ? "OK" : "BAD"); + rapidjson::Document root; + root.SetObject(); + root.AddMember("status", rapidjson::Value(status.c_str(), status.size()), root.GetAllocator()); + root.AddMember("msg", rapidjson::Value(msg.c_str(), msg.size()), root.GetAllocator()); + rapidjson::StringBuffer strbuf; + rapidjson::PrettyWriter writer(strbuf); + root.Accept(writer); + + req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str()); + HttpChannel::send_reply(req, HttpStatus::OK, strbuf.GetString()); +} + +} // namespace doris diff --git a/be/src/http/action/update_config_action.h b/be/src/http/action/update_config_action.h new file mode 100644 index 00000000000000..97965a218de59e --- /dev/null +++ b/be/src/http/action/update_config_action.h @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "http/http_handler.h" + +namespace doris { + +// Update BE config. +class UpdateConfigAction : public HttpHandler { +public: + UpdateConfigAction() {} + virtual ~UpdateConfigAction() {} + + void handle(HttpRequest* req) override; +}; + +} // namespace doris diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index cdc3a31638b609..77c8958607695e 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -68,7 +68,7 @@ void logs_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* ou void config_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) { (*output) << "

Configurations

"; (*output) << "
";
-    for (const auto& it : *(config::confmap)) {
+    for (const auto& it : *(config::full_conf_map)) {
         (*output) << it.first << "=" << it.second << std::endl;
     }
     (*output) << "
"; diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp index b1fdb66eff84dc..0f09384570ed26 100644 --- a/be/src/http/http_client.cpp +++ b/be/src/http/http_client.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include "common/config.h" #include "http/http_client.h" namespace doris { diff --git a/be/src/olap/olap_meta.cpp b/be/src/olap/olap_meta.cpp index d1329c58d22b6f..22f7d7e1f766ac 100755 --- a/be/src/olap/olap_meta.cpp +++ b/be/src/olap/olap_meta.cpp @@ -38,7 +38,6 @@ using rocksdb::ReadOptions; using rocksdb::WriteOptions; using rocksdb::Slice; using rocksdb::Iterator; -using rocksdb::Status; using rocksdb::kDefaultColumnFamilyName; using rocksdb::NewFixedPrefixTransform; @@ -78,7 +77,7 @@ OLAPStatus OlapMeta::init() { ColumnFamilyOptions meta_column_family; meta_column_family.prefix_extractor.reset(NewFixedPrefixTransform(PREFIX_LENGTH)); column_families.emplace_back(META_COLUMN_FAMILY, meta_column_family); - Status s = DB::Open(options, db_path, column_families, &_handles, &_db); + rocksdb::Status s = DB::Open(options, db_path, column_families, &_handles, &_db); if (!s.ok() || _db == nullptr) { LOG(WARNING) << "rocks db open failed, reason:" << s.ToString(); return OLAP_ERR_META_OPEN_DB; @@ -90,7 +89,7 @@ OLAPStatus OlapMeta::get(const int column_family_index, const std::string& key, DorisMetrics::meta_read_request_total.increment(1); rocksdb::ColumnFamilyHandle* handle = _handles[column_family_index]; int64_t duration_ns = 0; - Status s = Status::OK(); + rocksdb::Status s; { SCOPED_RAW_TIMER(&duration_ns); s = _db->Get(ReadOptions(), handle, Slice(key), value); @@ -109,7 +108,7 @@ OLAPStatus OlapMeta::put(const int column_family_index, const std::string& key, DorisMetrics::meta_write_request_total.increment(1); rocksdb::ColumnFamilyHandle* handle = _handles[column_family_index]; int64_t duration_ns = 0; - Status s = Status::OK(); + rocksdb::Status s; { SCOPED_RAW_TIMER(&duration_ns); WriteOptions write_options; @@ -127,7 +126,7 @@ OLAPStatus OlapMeta::put(const int column_family_index, const std::string& key, OLAPStatus OlapMeta::remove(const int column_family_index, const std::string& key) { DorisMetrics::meta_write_request_total.increment(1); rocksdb::ColumnFamilyHandle* handle = _handles[column_family_index]; - Status s = Status::OK(); + rocksdb::Status s; int64_t duration_ns = 0; { SCOPED_RAW_TIMER(&duration_ns); @@ -152,7 +151,7 @@ OLAPStatus OlapMeta::iterate(const int column_family_index, const std::string& p } else { it->Seek(prefix); } - Status status = it->status(); + rocksdb::Status status = it->status(); if (!status.ok()) { LOG(WARNING) << "rocksdb seek failed. reason:" << status.ToString(); return OLAP_ERR_META_ITERATOR; diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index ec60b0bef37055..6015f5ebccd08c 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -160,13 +160,13 @@ void* StorageEngine::_fd_cache_clean_callback(void* arg) { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif - uint32_t interval = config::file_descriptor_cache_clean_interval; - if (interval <= 0) { - OLAP_LOG_WARNING("config of file descriptor clean interval is illegal: [%d], " - "force set to 3600", interval); - interval = 3600; - } while (!_stop_bg_worker) { + int32_t interval = config::file_descriptor_cache_clean_interval; + if (interval <= 0) { + OLAP_LOG_WARNING("config of file descriptor clean interval is illegal: [%d], " + "force set to 3600", interval); + interval = 3600; + } SLEEP_IN_BG_WORKER(interval); _start_clean_fd_cache(); @@ -179,13 +179,6 @@ void* StorageEngine::_base_compaction_thread_callback(void* arg, DataDir* data_d #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif - uint32_t interval = config::base_compaction_check_interval_seconds; - if (interval <= 0) { - OLAP_LOG_WARNING("base compaction check interval config is illegal: [%d], " - "force set to 1", interval); - interval = 1; - } - //string last_base_compaction_fs; //TTabletId last_base_compaction_tablet_id = -1; while (!_stop_bg_worker) { @@ -197,6 +190,12 @@ void* StorageEngine::_base_compaction_thread_callback(void* arg, DataDir* data_d _perform_base_compaction(data_dir); } + int32_t interval = config::base_compaction_check_interval_seconds; + if (interval <= 0) { + OLAP_LOG_WARNING("base compaction check interval config is illegal: [%d], " + "force set to 1", interval); + interval = 1; + } SLEEP_IN_BG_WORKER(interval); } @@ -253,17 +252,15 @@ void* StorageEngine::_disk_stat_monitor_thread_callback(void* arg) { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif - - uint32_t interval = config::disk_stat_monitor_interval; - - if (interval <= 0) { - LOG(WARNING) << "disk_stat_monitor_interval config is illegal: " << interval - << ", force set to 1"; - interval = 1; - } - while (!_stop_bg_worker) { _start_disk_stat_monitor(); + + int32_t interval = config::disk_stat_monitor_interval; + if (interval <= 0) { + LOG(WARNING) << "disk_stat_monitor_interval config is illegal: " << interval + << ", force set to 1"; + interval = 1; + } SLEEP_IN_BG_WORKER(interval); } @@ -275,12 +272,6 @@ void* StorageEngine::_cumulative_compaction_thread_callback(void* arg, DataDir* ProfilerRegisterThread(); #endif LOG(INFO) << "try to start cumulative compaction process!"; - uint32_t interval = config::cumulative_compaction_check_interval_seconds; - if (interval <= 0) { - LOG(WARNING) << "cumulative compaction check interval config is illegal:" << interval - << "will be forced set to one"; - interval = 1; - } while (!_stop_bg_worker) { // must be here, because this thread is start on start and @@ -290,6 +281,13 @@ void* StorageEngine::_cumulative_compaction_thread_callback(void* arg, DataDir* if (!data_dir->reach_capacity_limit(0)) { _perform_cumulative_compaction(data_dir); } + + int32_t interval = config::cumulative_compaction_check_interval_seconds; + if (interval <= 0) { + LOG(WARNING) << "cumulative compaction check interval config is illegal:" << interval + << "will be forced set to one"; + interval = 1; + } SLEEP_IN_BG_WORKER(interval); } @@ -300,17 +298,15 @@ void* StorageEngine::_unused_rowset_monitor_thread_callback(void* arg) { #ifdef GOOGLE_PROFILER ProfilerRegisterThread(); #endif - - uint32_t interval = config::unused_rowset_monitor_interval; - - if (interval <= 0) { - LOG(WARNING) << "unused_rowset_monitor_interval config is illegal: " << interval - << ", force set to 1"; - interval = 1; - } - while (!_stop_bg_worker) { start_delete_unused_rowset(); + + int32_t interval = config::unused_rowset_monitor_interval; + if (interval <= 0) { + LOG(WARNING) << "unused_rowset_monitor_interval config is illegal: " << interval + << ", force set to 1"; + interval = 1; + } SLEEP_IN_BG_WORKER(interval); } @@ -325,17 +321,18 @@ void* StorageEngine::_path_gc_thread_callback(void* arg) { #endif LOG(INFO) << "try to start path gc thread!"; - uint32_t interval = config::path_gc_check_interval_second; - if (interval <= 0) { - LOG(WARNING) << "path gc thread check interval config is illegal:" << interval - << "will be forced set to half hour"; - interval = 1800; // 0.5 hour - } while (!_stop_bg_worker) { LOG(INFO) << "try to perform path gc!"; // perform path gc by rowset id ((DataDir*)arg)->perform_path_gc_by_rowsetid(); + + int32_t interval = config::path_gc_check_interval_second; + if (interval <= 0) { + LOG(WARNING) << "path gc thread check interval config is illegal:" << interval + << "will be forced set to half hour"; + interval = 1800; // 0.5 hour + } SLEEP_IN_BG_WORKER(interval); } @@ -348,16 +345,17 @@ void* StorageEngine::_path_scan_thread_callback(void* arg) { #endif LOG(INFO) << "try to start path scan thread!"; - uint32_t interval = config::path_scan_interval_second; - if (interval <= 0) { - LOG(WARNING) << "path gc thread check interval config is illegal:" << interval - << "will be forced set to one day"; - interval = 24 * 3600; // one day - } while (!_stop_bg_worker) { LOG(INFO) << "try to perform path scan!"; ((DataDir*)arg)->perform_path_scan(); + + int32_t interval = config::path_scan_interval_second; + if (interval <= 0) { + LOG(WARNING) << "path gc thread check interval config is illegal:" << interval + << "will be forced set to one day"; + interval = 24 * 3600; // one day + } SLEEP_IN_BG_WORKER(interval); } diff --git a/be/src/plugin/plugin_mgr.cpp b/be/src/plugin/plugin_mgr.cpp index fc459dad5649f7..295470a1bdb301 100644 --- a/be/src/plugin/plugin_mgr.cpp +++ b/be/src/plugin/plugin_mgr.cpp @@ -17,6 +17,7 @@ #include +#include "common/config.h" #include "plugin/plugin_mgr.h" #include "gutil/strings/substitute.h" @@ -159,4 +160,4 @@ Status PluginMgr::get_all_plugin_info(std::vector* plugin_info_list return Status::OK(); } -} \ No newline at end of file +} diff --git a/be/src/runtime/external_scan_context_mgr.cpp b/be/src/runtime/external_scan_context_mgr.cpp index 57ad940a605fe8..6583b0204fa0b3 100644 --- a/be/src/runtime/external_scan_context_mgr.cpp +++ b/be/src/runtime/external_scan_context_mgr.cpp @@ -27,7 +27,7 @@ namespace doris { -ExternalScanContextMgr::ExternalScanContextMgr(ExecEnv* exec_env) : _exec_env(exec_env), _is_stop(false), _scan_context_gc_interval_min(doris::config::scan_context_gc_interval_min) { +ExternalScanContextMgr::ExternalScanContextMgr(ExecEnv* exec_env) : _exec_env(exec_env), _is_stop(false) { // start the reaper thread for gc the expired context _keep_alive_reaper.reset( new std::thread( @@ -88,7 +88,7 @@ Status ExternalScanContextMgr::clear_scan_context(const std::string& context_id) void ExternalScanContextMgr::gc_expired_context() { while (!_is_stop) { - std::this_thread::sleep_for(std::chrono::seconds(_scan_context_gc_interval_min * 60)); + std::this_thread::sleep_for(std::chrono::seconds(doris::config::scan_context_gc_interval_min * 60)); time_t current_time = time(NULL); std::vector> expired_contexts; { diff --git a/be/src/runtime/external_scan_context_mgr.h b/be/src/runtime/external_scan_context_mgr.h index 691fdfc64b6a90..9730e55d191874 100644 --- a/be/src/runtime/external_scan_context_mgr.h +++ b/be/src/runtime/external_scan_context_mgr.h @@ -70,7 +70,6 @@ class ExternalScanContextMgr { bool _is_stop; std::unique_ptr _keep_alive_reaper; std::mutex _lock; - u_int32_t _scan_context_gc_interval_min; }; } \ No newline at end of file diff --git a/be/src/runtime/load_channel_mgr.cpp b/be/src/runtime/load_channel_mgr.cpp index a801f501ffbf3c..0e31bb10fac554 100644 --- a/be/src/runtime/load_channel_mgr.cpp +++ b/be/src/runtime/load_channel_mgr.cpp @@ -26,7 +26,7 @@ namespace doris { -// Calculate the totol memory limit of all load tasks on this BE +// Calculate the total memory limit of all load tasks on this BE static int64_t calc_process_max_load_memory(int64_t process_mem_limit) { if (process_mem_limit == -1) { // no limit diff --git a/be/src/runtime/mem_pool.h b/be/src/runtime/mem_pool.h index 2b05fb73dc0366..a9da9cf0dfe72b 100644 --- a/be/src/runtime/mem_pool.h +++ b/be/src/runtime/mem_pool.h @@ -25,6 +25,7 @@ #include #include +#include "common/config.h" #include "common/logging.h" #include "gutil/dynamic_annotations.h" #include "util/bit_util.h" diff --git a/be/src/runtime/thread_resource_mgr.cpp b/be/src/runtime/thread_resource_mgr.cpp index 3694f60f5ab041..4b3a6ac5c2c49e 100644 --- a/be/src/runtime/thread_resource_mgr.cpp +++ b/be/src/runtime/thread_resource_mgr.cpp @@ -21,6 +21,7 @@ #include +#include "common/config.h" #include "common/logging.h" #include "util/cpu_info.h" diff --git a/be/src/service/backend_options.cpp b/be/src/service/backend_options.cpp index 10c78674966c1f..6935920c461fae 100644 --- a/be/src/service/backend_options.cpp +++ b/be/src/service/backend_options.cpp @@ -21,6 +21,7 @@ #include "gutil/strings/split.h" +#include "common/config.h" #include "common/logging.h" #include "common/status.h" #include "util/network_util.h" diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index 0a2aeb4e2492bb..caf6a2c2658113 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -28,6 +28,7 @@ #include "http/action/restore_tablet_action.h" #include "http/action/snapshot_action.h" #include "http/action/stream_load.h" +#include "http/action/update_config_action.h" #include "http/default_path_handlers.h" #include "http/download_action.h" #include "http/ev_http_server.h" @@ -119,6 +120,9 @@ Status HttpService::start() { CompactionAction* run_compaction_action = new CompactionAction(CompactionActionType::RUN_COMPACTION); _ev_http_server->register_handler(HttpMethod::POST, "/api/compaction/run", run_compaction_action); + UpdateConfigAction* update_config_action = new UpdateConfigAction(); + _ev_http_server->register_handler(HttpMethod::POST, "/api/update_config", update_config_action); + RETURN_IF_ERROR(_ev_http_server->start()); return Status::OK(); } diff --git a/be/src/util/metrics.h b/be/src/util/metrics.h index c101b6c60dffa1..331c5c38717fef 100644 --- a/be/src/util/metrics.h +++ b/be/src/util/metrics.h @@ -26,6 +26,7 @@ #include #include +#include "common/config.h" #include "util/spinlock.h" #include "util/core_local.h" diff --git a/be/src/util/runtime_profile.cpp b/be/src/util/runtime_profile.cpp index e596064d89d050..51432f715f479b 100644 --- a/be/src/util/runtime_profile.cpp +++ b/be/src/util/runtime_profile.cpp @@ -23,6 +23,7 @@ #include #include +#include "common/config.h" #include "common/object_pool.h" #include "util/container_util.hpp" #include "util/cpu_info.h" diff --git a/be/src/util/thrift_rpc_helper.h b/be/src/util/thrift_rpc_helper.h index 0b014f5ddcb36d..543e43aa108ba1 100644 --- a/be/src/util/thrift_rpc_helper.h +++ b/be/src/util/thrift_rpc_helper.h @@ -17,6 +17,7 @@ #pragma once +#include "common/config.h" #include "common/status.h" #include "gen_cpp/FrontendService_types.h" diff --git a/be/test/common/config_test.cpp b/be/test/common/config_test.cpp index f17f0bff6a11f9..ba35e511a47560 100644 --- a/be/test/common/config_test.cpp +++ b/be/test/common/config_test.cpp @@ -20,6 +20,7 @@ #undef __IN_CONFIGBASE_CPP__ #include +#include "common/status.h" namespace doris { using namespace config; @@ -45,7 +46,7 @@ TEST_F(ConfigTest, DumpAllConfigs) { config::init(nullptr, true); std::stringstream ss; - for (const auto& it : *(config::confmap)) { + for (const auto& it : *(config::full_conf_map)) { ss << it.first << "=" << it.second << std::endl; } ASSERT_EQ( @@ -66,6 +67,78 @@ cfg_std_vector_std_string=doris, config, test, string ss.str()); } +TEST_F(ConfigTest, UpdateConfigs) { + CONF_Bool(cfg_bool_immutable, "true"); + CONF_mBool(cfg_bool, "false"); + CONF_mDouble(cfg_double, "123.456"); + CONF_mInt16(cfg_int16_t, "2561"); + CONF_mInt32(cfg_int32_t, "65536123"); + CONF_mInt64(cfg_int64_t, "4294967296123"); + CONF_String(cfg_std_string, "doris_config_test_string"); + + config::init(nullptr, true); + + // bool + ASSERT_FALSE(cfg_bool); + ASSERT_TRUE(config::set_config("cfg_bool", "true").ok()); + ASSERT_TRUE(cfg_bool); + + // double + ASSERT_EQ(cfg_double, 123.456); + ASSERT_TRUE(config::set_config("cfg_double", "654.321").ok()); + ASSERT_EQ(cfg_double, 654.321); + + // int16 + ASSERT_EQ(cfg_int16_t, 2561); + ASSERT_TRUE(config::set_config("cfg_int16_t", "2562").ok()); + ASSERT_EQ(cfg_int16_t, 2562); + + // int32 + ASSERT_EQ(cfg_int32_t, 65536123); + ASSERT_TRUE(config::set_config("cfg_int32_t", "65536124").ok()); + ASSERT_EQ(cfg_int32_t, 65536124); + + // int64 + ASSERT_EQ(cfg_int64_t, 4294967296123); + ASSERT_TRUE(config::set_config("cfg_int64_t", "4294967296124").ok()); + ASSERT_EQ(cfg_int64_t, 4294967296124); + + // not exist + Status s = config::set_config("cfg_not_exist", "123"); + ASSERT_FALSE(s.ok()); + ASSERT_EQ(s.to_string(), "Not found: 'cfg_not_exist' is not found"); + + // immutable + ASSERT_TRUE(cfg_bool_immutable); + s = config::set_config("cfg_bool_immutable", "false"); + ASSERT_FALSE(s.ok()); + ASSERT_EQ(s.to_string(), "Not supported: 'cfg_bool_immutable' is not support to modify"); + ASSERT_TRUE(cfg_bool_immutable); + + // convert error + s = config::set_config("cfg_bool", "falseeee"); + ASSERT_FALSE(s.ok()); + ASSERT_EQ(s.to_string(), "Invalid argument: convert 'falseeee' as bool failed"); + ASSERT_TRUE(cfg_bool); + + s = config::set_config("cfg_double", ""); + ASSERT_FALSE(s.ok()); + ASSERT_EQ(s.to_string(), "Invalid argument: convert '' as double failed"); + ASSERT_EQ(cfg_double, 654.321); + + // convert error + s = config::set_config("cfg_int32_t", "4294967296124"); + ASSERT_FALSE(s.ok()); + ASSERT_EQ(s.to_string(), "Invalid argument: convert '4294967296124' as int32_t failed"); + ASSERT_EQ(cfg_int32_t, 65536124); + + // not support + s = config::set_config("cfg_std_string", "test"); + ASSERT_FALSE(s.ok()); + ASSERT_EQ(s.to_string(), "Not supported: 'cfg_std_string' is not support to modify"); + ASSERT_EQ(cfg_std_string, "doris_config_test_string"); +} + } // namespace doris int main(int argc, char** argv) { diff --git a/be/test/common/resource_tls_test.cpp b/be/test/common/resource_tls_test.cpp index 567d324c950f12..e0e22507d5ac97 100644 --- a/be/test/common/resource_tls_test.cpp +++ b/be/test/common/resource_tls_test.cpp @@ -20,6 +20,7 @@ #include #include "gen_cpp/Types_types.h" +#include "common/configbase.h" #include "util/logging.h" namespace doris { diff --git a/be/test/exec/tablet_sink_test.cpp b/be/test/exec/tablet_sink_test.cpp index aff3653a3a46a9..2f8680a30bd971 100644 --- a/be/test/exec/tablet_sink_test.cpp +++ b/be/test/exec/tablet_sink_test.cpp @@ -55,8 +55,6 @@ class OlapTableSinkTest : public testing::Test { _env->_load_stream_mgr = new LoadStreamMgr(); _env->_brpc_stub_cache = new BrpcStubCache(); _env->_buffer_reservation = new ReservationTracker(); - - config::tablet_writer_open_rpc_timeout_sec = 60; } void TearDown() override { delete _env->_brpc_stub_cache; diff --git a/be/test/exprs/hybird_set_test.cpp b/be/test/exprs/hybird_set_test.cpp index 53d1c32099182a..f8a102ccb0a910 100644 --- a/be/test/exprs/hybird_set_test.cpp +++ b/be/test/exprs/hybird_set_test.cpp @@ -19,6 +19,7 @@ #include #include +#include "common/configbase.h" #include "util/logging.h" namespace doris { diff --git a/be/test/olap/bloom_filter_test.cpp b/be/test/olap/bloom_filter_test.cpp index 42e689b990e31d..c77dff07447f68 100644 --- a/be/test/olap/bloom_filter_test.cpp +++ b/be/test/olap/bloom_filter_test.cpp @@ -20,6 +20,7 @@ #include #include "olap/bloom_filter.hpp" +#include "common/configbase.h" #include "util/logging.h" using std::string; diff --git a/be/test/olap/byte_buffer_test.cpp b/be/test/olap/byte_buffer_test.cpp index 80a9fb567451e9..641b990b5010cd 100755 --- a/be/test/olap/byte_buffer_test.cpp +++ b/be/test/olap/byte_buffer_test.cpp @@ -21,6 +21,7 @@ #include "boost/filesystem.hpp" #include "olap/byte_buffer.h" #include "olap/file_helper.h" +#include "common/configbase.h" #include "util/logging.h" namespace doris { diff --git a/be/test/olap/file_helper_test.cpp b/be/test/olap/file_helper_test.cpp index 90d8e46aceca87..667e7db6e24b8c 100644 --- a/be/test/olap/file_helper_test.cpp +++ b/be/test/olap/file_helper_test.cpp @@ -23,6 +23,7 @@ #include "olap/olap_define.h" #include "olap/file_helper.h" #include "boost/filesystem.hpp" +#include "common/configbase.h" #include "util/logging.h" #ifndef BE_TEST diff --git a/be/test/olap/file_utils_test.cpp b/be/test/olap/file_utils_test.cpp index 2c18f1d6175445..0a1757333984f1 100644 --- a/be/test/olap/file_utils_test.cpp +++ b/be/test/olap/file_utils_test.cpp @@ -28,6 +28,7 @@ #include "util/file_utils.h" #include "util/logging.h" #include "env/env.h" +#include "common/configbase.h" #ifndef BE_TEST #define BE_TEST diff --git a/be/test/runtime/large_int_value_test.cpp b/be/test/runtime/large_int_value_test.cpp index ffbec1a31b13d5..37771458f4fae7 100644 --- a/be/test/runtime/large_int_value_test.cpp +++ b/be/test/runtime/large_int_value_test.cpp @@ -23,6 +23,7 @@ #include #include +#include "common/configbase.h" #include "common/logging.h" namespace doris { diff --git a/be/test/util/cidr_test.cpp b/be/test/util/cidr_test.cpp index 3db83d005b8475..6b562d3e7e6f8e 100644 --- a/be/test/util/cidr_test.cpp +++ b/be/test/util/cidr_test.cpp @@ -23,6 +23,7 @@ #include +#include "common/configbase.h" #include "util/cpu_info.h" #include "util/logging.h" diff --git a/be/test/util/filesystem_util_test.cpp b/be/test/util/filesystem_util_test.cpp index ab38439ab8e1b7..458f8a93839229 100644 --- a/be/test/util/filesystem_util_test.cpp +++ b/be/test/util/filesystem_util_test.cpp @@ -22,6 +22,7 @@ #include #include +#include "common/configbase.h" #include "util/logging.h" namespace doris { diff --git a/be/test/util/internal_queue_test.cpp b/be/test/util/internal_queue_test.cpp index 735c90c2c1ff4e..ec82655daf5457 100644 --- a/be/test/util/internal_queue_test.cpp +++ b/be/test/util/internal_queue_test.cpp @@ -22,6 +22,7 @@ #include #include +#include "common/configbase.h" #include "util/logging.h" using std::vector; From 175eb7894abab92a04f33f36497fec7b15db233c Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Sun, 5 Apr 2020 02:36:02 +0000 Subject: [PATCH 2/5] revert tablet_sink_test.cpp --- be/test/exec/tablet_sink_test.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/be/test/exec/tablet_sink_test.cpp b/be/test/exec/tablet_sink_test.cpp index 2f8680a30bd971..aff3653a3a46a9 100644 --- a/be/test/exec/tablet_sink_test.cpp +++ b/be/test/exec/tablet_sink_test.cpp @@ -55,6 +55,8 @@ class OlapTableSinkTest : public testing::Test { _env->_load_stream_mgr = new LoadStreamMgr(); _env->_brpc_stub_cache = new BrpcStubCache(); _env->_buffer_reservation = new ReservationTracker(); + + config::tablet_writer_open_rpc_timeout_sec = 60; } void TearDown() override { delete _env->_brpc_stub_cache; From d602d0a273339fcb34c5b4345cb575d3b455e954 Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Tue, 7 Apr 2020 11:33:51 +0800 Subject: [PATCH 3/5] fix --- be/src/http/action/update_config_action.cpp | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/be/src/http/action/update_config_action.cpp b/be/src/http/action/update_config_action.cpp index 37d5ed5d7456b9..87baf2dbf05f6f 100644 --- a/be/src/http/action/update_config_action.cpp +++ b/be/src/http/action/update_config_action.cpp @@ -41,26 +41,26 @@ const static std::string HEADER_JSON = "application/json"; void UpdateConfigAction::handle(HttpRequest* req) { LOG(INFO) << req->debug_string(); - bool success = true; + Status s; std::string msg; if (req->params()->size() != 1) { - success = false; - msg = "Now only support to set a single config once, via 'config_name=new_value'."; + s = Status::InvalidArgument(); + msg = "Now only support to set a single config once, via 'config_name=new_value'"; } else { DCHECK(req->params()->size() == 1); - for (const auto& it : *req->params()) { - Status s = config::set_config(it.first, it.second); - success = s.ok(); - LOG(WARNING) << "set_config " << it.first << "=" << it.second - << (success ? " success" : " failed"); - if (!success) { - msg = strings::Substitute("set $0=$1 failed, reason: $2", it.first, it.second, - s.to_string()); - } + const std::string& config = req->params()->begin().first; + const std::string& new_value = req->params()->begin().second; + s = config::set_config(config, new_value); + if (s.ok()) { + LOG(INFO) << "set_config " << config << "=" << new_value << " success"; + } else { + LOG(WARNING) << "set_config " << config << "=" << new_value << " failed"; + msg = strings::Substitute("set $0=$1 failed, reason: $2", config, new_value, + s.to_string()); } } - std::string status(success ? "OK" : "BAD"); + std::string status(s.ok() ? "OK" : "BAD"); rapidjson::Document root; root.SetObject(); root.AddMember("status", rapidjson::Value(status.c_str(), status.size()), root.GetAllocator()); From f04f279649f2eb06ac4e2bbdad13fc27cba787d1 Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Tue, 7 Apr 2020 11:35:42 +0800 Subject: [PATCH 4/5] rename _s_field_map --- be/src/common/configbase.cpp | 8 ++++---- be/src/common/configbase.h | 8 ++++---- be/test/common/config_test.cpp | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/be/src/common/configbase.cpp b/be/src/common/configbase.cpp index 9a8d8b633f2081..115c2db855e952 100644 --- a/be/src/common/configbase.cpp +++ b/be/src/common/configbase.cpp @@ -34,7 +34,7 @@ namespace doris { namespace config { -std::map* Register::_s_fieldlist = nullptr; +std::map* Register::_s_field_map = nullptr; std::map* full_conf_map = nullptr; Properties props; @@ -275,7 +275,7 @@ bool init(const char* filename, bool fillconfmap) { } // set conf fields - for (const auto& it : *Register::_s_fieldlist) { + for (const auto& it : *Register::_s_field_map) { SET_FIELD(it.second, bool, fillconfmap); SET_FIELD(it.second, int16_t, fillconfmap); SET_FIELD(it.second, int32_t, fillconfmap); @@ -308,8 +308,8 @@ bool init(const char* filename, bool fillconfmap) { } Status set_config(const std::string& field, const std::string& value) { - auto it = Register::_s_fieldlist->find(field); - if (it == Register::_s_fieldlist->end()) { + auto it = Register::_s_field_map->find(field); + if (it == Register::_s_field_map->end()) { return Status::NotFound(strings::Substitute("'$0' is not found", field)); } diff --git a/be/src/common/configbase.h b/be/src/common/configbase.h index 64f6aa759e26c9..9dd58bb7b62071 100644 --- a/be/src/common/configbase.h +++ b/be/src/common/configbase.h @@ -47,16 +47,16 @@ class Register { }; public: - static std::map* _s_fieldlist; + static std::map* _s_field_map; public: Register(const char* ftype, const char* fname, void* fstorage, const char* fdefval, bool fvalmutable) { - if (_s_fieldlist == nullptr) { - _s_fieldlist = new std::map(); + if (_s_field_map == nullptr) { + _s_field_map = new std::map(); } Field field(ftype, fname, fstorage, fdefval, fvalmutable); - _s_fieldlist->insert(std::make_pair(std::string(fname), field)); + _s_field_map->insert(std::make_pair(std::string(fname), field)); } }; diff --git a/be/test/common/config_test.cpp b/be/test/common/config_test.cpp index ba35e511a47560..a6d36ae01c5d96 100644 --- a/be/test/common/config_test.cpp +++ b/be/test/common/config_test.cpp @@ -26,7 +26,7 @@ namespace doris { using namespace config; class ConfigTest : public testing::Test { - void SetUp() override { config::Register::_s_fieldlist->clear(); } + void SetUp() override { config::Register::_s_field_map->clear(); } }; TEST_F(ConfigTest, DumpAllConfigs) { From 1a9b75396c7ec9b3ed45e475da3c18ef28462aa6 Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Tue, 7 Apr 2020 03:48:39 +0000 Subject: [PATCH 5/5] fix --- be/src/http/action/update_config_action.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/be/src/http/action/update_config_action.cpp b/be/src/http/action/update_config_action.cpp index 87baf2dbf05f6f..91eb6b0df00b2e 100644 --- a/be/src/http/action/update_config_action.cpp +++ b/be/src/http/action/update_config_action.cpp @@ -44,12 +44,12 @@ void UpdateConfigAction::handle(HttpRequest* req) { Status s; std::string msg; if (req->params()->size() != 1) { - s = Status::InvalidArgument(); + s = Status::InvalidArgument(""); msg = "Now only support to set a single config once, via 'config_name=new_value'"; } else { DCHECK(req->params()->size() == 1); - const std::string& config = req->params()->begin().first; - const std::string& new_value = req->params()->begin().second; + const std::string& config = req->params()->begin()->first; + const std::string& new_value = req->params()->begin()->second; s = config::set_config(config, new_value); if (s.ok()) { LOG(INFO) << "set_config " << config << "=" << new_value << " success";