Skip to content

[config] Support to modify configs when BE is running without restarting #3264

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
imay merged 5 commits into from
Apr 8, 2020
Merged

[config] Support to modify configs when BE is running without restarting #3264

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5bb7b92
[config] Support to modify configs when BE is running without restart…
acelyc111 Mar 31, 2020
175eb78
revert tablet_sink_test.cpp
acelyc111 Apr 5, 2020
d602d0a
fix
acelyc111 Apr 7, 2020
f04f279
rename _s_field_map
acelyc111 Apr 7, 2020
1a9b753
fix
acelyc111 Apr 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 86 additions & 86 deletions be/src/common/config.h
View file
Open in desktop

Large diffs are not rendered by default.

261 changes: 157 additions & 104 deletions be/src/common/configbase.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,91 +28,19 @@
#include "common/config.h"
#undef __IN_CONFIGBASE_CPP__

#include "common/status.h"
#include "gutil/strings/substitute.h"

namespace doris {
namespace config {

std::list<Register::Field>* Register::_s_fieldlist = nullptr;
std::map<std::string, std::string>* confmap = nullptr;
std::map<std::string, Register::Field>* Register::_s_field_map = nullptr;
std::map<std::string, std::string>* full_conf_map = nullptr;

Properties props;

// load conf file
bool Properties::load(const char* filename) {
// if filename is null, use the empty props
if (filename == nullptr) {
return true;
}

// open the conf file
std::ifstream input(filename);
if (!input.is_open()) {
std::cerr << "config::load() failed to open the file:" << filename << std::endl;
return false;
}

// load properties
std::string line;
std::string key;
std::string value;
line.reserve(512);
while (input) {
// read one line at a time
std::getline(input, line);

// remove left and right spaces
trim(line);

// ignore comments
if (line.empty() || line[0] == '#') {
continue;
}

// read key and value
splitkv(line, key, value);
trim(key);
trim(value);

// insert into propmap
propmap[key] = value;
}

// close the conf file
input.close();

return true;
}

template <typename T>
bool Properties::get(const char* key, const char* defstr, T& retval) const {
std::map<std::string, std::string>::const_iterator it = propmap.find(std::string(key));
std::string valstr = it != propmap.end() ? it->second : std::string(defstr);
trim(valstr);
if (!replaceenv(valstr)) {
return false;
}
return strtox(valstr, retval);
}

template <typename T>
bool Properties::strtox(const std::string& valstr, std::vector<T>& retval) {
std::stringstream ss(valstr);
std::string item;
T t;
while (std::getline(ss, item, ',')) {
if (!strtox(trim(item), t)) {
return false;
}
retval.push_back(t);
}
return true;
}

const std::map<std::string, std::string>& Properties::getmap() const {
return propmap;
}

// trim string
std::string& Properties::trim(std::string& s) {
std::string& trim(std::string& s) {
// rtrim
s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun<int, int>(std::isspace)))
.base(),
Expand All @@ -124,7 +52,7 @@ std::string& Properties::trim(std::string& s) {
}

// split string by '='
void Properties::splitkv(const std::string& s, std::string& k, std::string& v) {
void splitkv(const std::string& s, std::string& k, std::string& v) {
const char sep = '=';
int start = 0;
int end = 0;
Expand All @@ -138,7 +66,7 @@ void Properties::splitkv(const std::string& s, std::string& k, std::string& v) {
}

// replace env variables
bool Properties::replaceenv(std::string& s) {
bool replaceenv(std::string& s) {
std::size_t pos = 0;
std::size_t start = 0;
while ((start = s.find("${", pos)) != std::string::npos) {
Expand All @@ -158,7 +86,28 @@ bool Properties::replaceenv(std::string& s) {
return true;
}

bool Properties::strtox(const std::string& valstr, bool& retval) {
bool strtox(const std::string& valstr, bool& retval);
bool strtox(const std::string& valstr, int16_t& retval);
bool strtox(const std::string& valstr, int32_t& retval);
bool strtox(const std::string& valstr, int64_t& retval);
bool strtox(const std::string& valstr, double& retval);
bool strtox(const std::string& valstr, std::string& retval);

template <typename T>
bool strtox(const std::string& valstr, std::vector<T>& retval) {
std::stringstream ss(valstr);
std::string item;
T t;
while (std::getline(ss, item, ',')) {
if (!strtox(trim(item), t)) {
return false;
}
retval.push_back(t);
}
return true;
}

bool strtox(const std::string& valstr, bool& retval) {
if (valstr.compare("true") == 0) {
retval = true;
} else if (valstr.compare("false") == 0) {
Expand All @@ -170,7 +119,7 @@ bool Properties::strtox(const std::string& valstr, bool& retval) {
}

template <typename T>
bool Properties::strtointeger(const std::string& valstr, T& retval) {
bool strtointeger(const std::string& valstr, T& retval) {
if (valstr.length() == 0) {
return false; // empty-string is only allowed for string type.
}
Expand All @@ -181,26 +130,28 @@ bool Properties::strtointeger(const std::string& valstr, T& retval) {
if (errno || end != valcstr + strlen(valcstr)) {
return false; // bad parse
}
T tmp = retval;
retval = static_cast<T>(ret64);
if (retval != ret64) {
retval = tmp;
return false;
}
return true;
}

bool Properties::strtox(const std::string& valstr, int16_t& retval) {
bool strtox(const std::string& valstr, int16_t& retval) {
return strtointeger(valstr, retval);
}

bool Properties::strtox(const std::string& valstr, int32_t& retval) {
bool strtox(const std::string& valstr, int32_t& retval) {
return strtointeger(valstr, retval);
}

bool Properties::strtox(const std::string& valstr, int64_t& retval) {
bool strtox(const std::string& valstr, int64_t& retval) {
return strtointeger(valstr, retval);
}

bool Properties::strtox(const std::string& valstr, double& retval) {
bool strtox(const std::string& valstr, double& retval) {
if (valstr.length() == 0) {
return false; // empty-string is only allowed for string type.
}
Expand All @@ -214,11 +165,78 @@ bool Properties::strtox(const std::string& valstr, double& retval) {
return true;
}

bool Properties::strtox(const std::string& valstr, std::string& retval) {
bool strtox(const std::string& valstr, std::string& retval) {
retval = valstr;
return true;
}

// load conf file
bool Properties::load(const char* filename) {
// if filename is null, use the empty props
if (filename == nullptr) {
return true;
}

// open the conf file
std::ifstream input(filename);
if (!input.is_open()) {
std::cerr << "config::load() failed to open the file:" << filename << std::endl;
return false;
}

// load properties
std::string line;
std::string key;
std::string value;
line.reserve(512);
while (input) {
// read one line at a time
std::getline(input, line);

// remove left and right spaces
trim(line);

// ignore comments
if (line.empty() || line[0] == '#') {
continue;
}

// read key and value
splitkv(line, key, value);
trim(key);
trim(value);

// insert into file_conf_map
file_conf_map[key] = value;
}

// close the conf file
input.close();

return true;
}

template <typename T>
bool Properties::get(const char* key, const char* defstr, T& retval) const {
const auto& it = file_conf_map.find(std::string(key));
std::string valstr = it != file_conf_map.end() ? it->second : std::string(defstr);
trim(valstr);
if (!replaceenv(valstr)) {
return false;
}
return strtox(valstr, retval);
}

template <typename T>
bool update(const std::string& value, T& retval) {
std::string valstr(value);
trim(valstr);
if (!replaceenv(valstr)) {
return false;
}
return strtox(valstr, retval);
}

template <typename T>
std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
size_t last = v.size() - 1;
Expand All @@ -240,7 +258,7 @@ std::ostream& operator<<(std::ostream& out, const std::vector<T>& v) {
if (FILL_CONFMAP) { \
std::ostringstream oss; \
oss << (*reinterpret_cast<TYPE*>((FIELD).storage)); \
(*confmap)[(FIELD).name] = oss.str(); \
(*full_conf_map)[(FIELD).name] = oss.str(); \
} \
continue; \
}
Expand All @@ -251,29 +269,64 @@ bool init(const char* filename, bool fillconfmap) {
if (!props.load(filename)) {
return false;
}
// fill confmap ?
if (fillconfmap && confmap == nullptr) {
confmap = new std::map<std::string, std::string>();
// fill full_conf_map ?
if (fillconfmap && full_conf_map == nullptr) {
full_conf_map = new std::map<std::string, std::string>();
}

// set conf fields
for (const auto& it : *Register::_s_fieldlist) {
SET_FIELD(it, bool, fillconfmap);
SET_FIELD(it, int16_t, fillconfmap);
SET_FIELD(it, int32_t, fillconfmap);
SET_FIELD(it, int64_t, fillconfmap);
SET_FIELD(it, double, fillconfmap);
SET_FIELD(it, std::string, fillconfmap);
SET_FIELD(it, std::vector<bool>, fillconfmap);
SET_FIELD(it, std::vector<int16_t>, fillconfmap);
SET_FIELD(it, std::vector<int32_t>, fillconfmap);
SET_FIELD(it, std::vector<int64_t>, fillconfmap);
SET_FIELD(it, std::vector<double>, fillconfmap);
SET_FIELD(it, std::vector<std::string>, fillconfmap);
for (const auto& it : *Register::_s_field_map) {
SET_FIELD(it.second, bool, fillconfmap);
SET_FIELD(it.second, int16_t, fillconfmap);
SET_FIELD(it.second, int32_t, fillconfmap);
SET_FIELD(it.second, int64_t, fillconfmap);
SET_FIELD(it.second, double, fillconfmap);
SET_FIELD(it.second, std::string, fillconfmap);
SET_FIELD(it.second, std::vector<bool>, fillconfmap);
SET_FIELD(it.second, std::vector<int16_t>, fillconfmap);
SET_FIELD(it.second, std::vector<int32_t>, fillconfmap);
SET_FIELD(it.second, std::vector<int64_t>, fillconfmap);
SET_FIELD(it.second, std::vector<double>, fillconfmap);
SET_FIELD(it.second, std::vector<std::string>, fillconfmap);
}

return true;
}

#define UPDATE_FIELD(FIELD, VALUE, TYPE) \
if (strcmp((FIELD).type, #TYPE) == 0) { \
if (!update((VALUE), *reinterpret_cast<TYPE*>((FIELD).storage))) { \
return Status::InvalidArgument( \
strings::Substitute("convert '$0' as $1 failed", VALUE, #TYPE)); \
} \
if (full_conf_map != nullptr) { \
std::ostringstream oss; \
oss << (*reinterpret_cast<TYPE*>((FIELD).storage)); \
(*full_conf_map)[(FIELD).name] = oss.str(); \
} \
return Status::OK(); \
}

Status set_config(const std::string& field, const std::string& value) {
auto it = Register::_s_field_map->find(field);
if (it == Register::_s_field_map->end()) {
return Status::NotFound(strings::Substitute("'$0' is not found", field));
}

if (!it->second.valmutable) {
return Status::NotSupported(strings::Substitute("'$0' is not support to modify", field));
}

UPDATE_FIELD(it->second, value, bool);
UPDATE_FIELD(it->second, value, int16_t);
UPDATE_FIELD(it->second, value, int32_t);
UPDATE_FIELD(it->second, value, int64_t);
UPDATE_FIELD(it->second, value, double);

// The other types are not thread safe to change dynamically.
return Status::NotSupported(strings::Substitute(
"'$0' is type of '$1' which is not support to modify", field, it->second.type));
}

} // namespace config
} // namespace doris
Loading