From 3832eea35caaffdcb95ef33944e3d3f9a20ffe7c Mon Sep 17 00:00:00 2001 From: huangkangping Date: Tue, 18 Jun 2019 19:36:15 +0800 Subject: [PATCH 1/3] add meta tool --- be/CMakeLists.txt | 2 +- be/src/olap/tablet_meta_manager.cpp | 18 ++- be/src/tools/meta_tool.cpp | 233 ++++++++++++++-------------- 3 files changed, 130 insertions(+), 123 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 61b3849737fb4b..603cd0593235bc 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -564,7 +564,7 @@ add_subdirectory(${SRC_DIR}/exprs) add_subdirectory(${SRC_DIR}/udf) add_subdirectory(${SRC_DIR}/runtime) add_subdirectory(${SRC_DIR}/testutil) -#add_subdirectory(${SRC_DIR}/tools) +add_subdirectory(${SRC_DIR}/tools) add_subdirectory(${SRC_DIR}/udf_samples) add_subdirectory(${SRC_DIR}/geo) diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp index bc9ea748caf5c0..f6cbddbc389919 100755 --- a/be/src/olap/tablet_meta_manager.cpp +++ b/be/src/olap/tablet_meta_manager.cpp @@ -86,9 +86,9 @@ OLAPStatus TabletMetaManager::save(DataDir* store, std::string value; tablet_meta->serialize(&value); OlapMeta* meta = store->get_meta(); - LOG(INFO) << "save tablet meta " - << " tablet_id=" << tablet_id - << " schema_hash=" << schema_hash; + LOG(INFO) << "save tablet meta" + << ", key:" << key + << ", meta length:" << value.length(); return meta->put(META_COLUMN_FAMILY_INDEX, key, value); } @@ -98,6 +98,7 @@ OLAPStatus TabletMetaManager::save(DataDir* store, key_stream << header_prefix << tablet_id << "_" << schema_hash; std::string key = key_stream.str(); VLOG(3) << "save tablet meta to meta store: key = " << key; + std::cout << "save tablet meta to meta store: key = " << key << std::endl; OlapMeta* meta = store->get_meta(); TabletMetaPB de_tablet_meta_pb; @@ -107,8 +108,7 @@ OLAPStatus TabletMetaManager::save(DataDir* store, } LOG(INFO) << "save tablet meta " - << " tablet_id=" << tablet_id - << " schema_hash=" << schema_hash + << ", key:" << key << " meta_size=" << meta_binary.length(); return meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary); } @@ -145,22 +145,26 @@ OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta, OLAPStatus TabletMetaManager::load_json_header(DataDir* store, const std::string& header_path) { std::ifstream infile(header_path); - char buffer[1024]; + char buffer[100000]; std::string json_header; + std::cout << "start to read json file" << std::endl; while (!infile.eof()) { - infile.getline(buffer, 1024); + infile.getline(buffer, 100000); json_header = json_header + buffer; } + std::cout << "json_header:" << json_header << std::endl; boost::algorithm::trim(json_header); TabletMetaPB tablet_meta_pb; bool ret = json2pb::JsonToProtoMessage(json_header, &tablet_meta_pb); if (!ret) { return OLAP_ERR_HEADER_LOAD_JSON_HEADER; } + std::string meta_binary; tablet_meta_pb.SerializeToString(&meta_binary); TTabletId tablet_id = tablet_meta_pb.tablet_id(); TSchemaHash schema_hash = tablet_meta_pb.schema_hash(); + std::cout << "start to save meta" << std::endl; return save(store, tablet_id, schema_hash, meta_binary); } diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index 49f9f54ef872fc..ed851d6d210e9f 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include #include #include -#include -#include -#include +#include +#include +#include #include #include "common/status.h" @@ -29,144 +28,148 @@ #include "olap/tablet_meta_manager.h" #include "olap/olap_define.h" #include "olap/tablet_meta.h" -#include "olap/olap_meta.h" #include "olap/utils.h" #include "json2pb/pb_to_json.h" +using boost::filesystem::canonical; +using boost::filesystem::path; using doris::DataDir; +using doris::OLAP_SUCCESS; using doris::OlapMeta; -using doris::TabletMetaManager; -using doris::TabletMeta; using doris::OLAPStatus; -using doris::OLAP_SUCCESS; using doris::Status; +using doris::TabletMeta; +using doris::TabletMetaManager; -const std::string HEADER_PREFIX = "hdr_"; +const std::string HEADER_PREFIX = "tabletmeta_"; -DEFINE_string(root_path, "./", "storage root path"); -DEFINE_string(operation, "get_header", - "valid operation: get_header, flag, load_header, delete_header, rollback, show_header"); -DEFINE_int64(tablet_id, 0, "tablet_id for header operation"); -DEFINE_int32(schema_hash, 0, "schema_hash for header operation"); -DEFINE_string(json_header_path, "", "json header file path"); -DEFINE_string(pb_header_path, "", "pb header file path"); +DECLARE_bool(help); +DEFINE_string(root_path, "", "storage root path"); +DEFINE_string(operation, "get_meta", + "valid operation: get_meta, flag, load_meta, delete_meta, show_meta"); +DEFINE_int64(tablet_id, 0, "tablet_id for tablet meta"); +DEFINE_int32(schema_hash, 0, "schema_hash for tablet meta"); +DEFINE_string(json_meta_path, "", "json meta file path"); +DEFINE_string(pb_meta_path, "", "pb meta file path"); -void print_usage(std::string progname) { - std::cout << progname << " is the Doris File tool." << std::endl; - std::cout << "Usage:" << std::endl; - std::cout << "./meta_tool --operation=get_header --tablet_id=tabletid --schema_hash=schemahash" << std::endl; - std::cout << "./meta_tool --operation=flag" << std::endl; - std::cout << "./meta_tool --operation=load_header --json_header_path=path" << std::endl; - std::cout << "./meta_tool --operation=delete_header --tablet_id=tabletid --schema_hash=schemahash" << std::endl; - std::cout << "./meta_tool --root_path=rootpath --operation=rollback" << std::endl; - std::cout << "./meta_tool --operation=show_header --pb_header_path=path" << std::endl; +std::string get_usage(std::string progname) { + std::stringstream ss; + ss << progname << " is the Doris BE Meta tool.\n"; + ss << "Stop BE first before use this tool.\n"; + ss << "Usage:\n"; + ss << "./meta_tool --operation=get_meta --root_path=/path/to/storage/path --tablet_id=tabletid --schema_hash=schemahash\n"; + ss << "./meta_tool --operation=load_meta --root_path=/path/to/storage/path --json_meta_path=path\n"; + ss << "./meta_tool --operation=delete_meta --root_path=/path/to/storage/path --tablet_id=tabletid --schema_hash=schemahash\n"; + ss << "./meta_tool --operation=show_meta --pb_meta_path=path\n"; + return ss.str(); } -int main(int argc, char** argv) { - google::ParseCommandLineFlags(&argc, &argv, true); +void show_meta() { + TabletMeta tablet_meta; + OLAPStatus s = tablet_meta.create_from_file(FLAGS_pb_meta_path); + if (s != OLAP_SUCCESS){ + std::cout << "load pb meta file:" << FLAGS_pb_meta_path << " failed" + << ", status:" << s << std::endl; + return; + } + std::string json_meta; + json2pb::Pb2JsonOptions json_options; + json_options.pretty_json = true; + doris::TabletMetaPB tablet_meta_pb; + tablet_meta.to_meta_pb(&tablet_meta_pb); + json2pb::ProtoMessageToJson(tablet_meta_pb, &json_meta, json_options); + std::cout << "tablet meta:" << std::endl; + std::cout << json_meta << std::endl; +} - std::string root_path = FLAGS_root_path; - if (FLAGS_root_path == "") { - std::cout << "empty root path" << std::endl; - print_usage(argv[0]); - return -1; - } else if (FLAGS_root_path.find("/") != 0) { - // relative path - char dir[PATH_MAX] = {0}; - readlink("/proc/self/exe", dir, PATH_MAX); - std::string path_prefix(dir); - path_prefix = path_prefix.substr(0, path_prefix.rfind("/") + 1); - std::string root_path_postfix = FLAGS_root_path; - // trim tailing / - if (root_path_postfix.rfind("/") == (root_path_postfix.size() -1)) { - root_path_postfix = root_path_postfix.substr(0, root_path_postfix.size() -1); - } +void get_meta(DataDir *data_dir) { + std::string value; + OLAPStatus s = TabletMetaManager::get_json_header(data_dir, FLAGS_tablet_id, FLAGS_schema_hash, &value); + if (s == doris::OLAP_ERR_META_KEY_NOT_FOUND) { + std::cout << "no tablet meta for tablet_id:" << FLAGS_tablet_id + << ", schema_hash:" << FLAGS_schema_hash << std::endl; + return; + } + std::cout << value << std::endl; +} - root_path = path_prefix + root_path_postfix; +void load_meta(DataDir *data_dir) { + // load json tablet meta into meta + OLAPStatus s = TabletMetaManager::load_json_header(data_dir, FLAGS_json_meta_path); + if (s != OLAP_SUCCESS) { + std::cout << "load meta failed, status:" << s << std::endl; + return; } - std::unique_ptr store(new(std::nothrow) DataDir(root_path)); - if (store.get() == NULL) { - std::cout << "new store failed" << std::endl; - return -1; + std::cout << "load meta successfully" << std::endl; +} + +void delete_meta(DataDir *data_dir) { + OLAPStatus s = TabletMetaManager::remove(data_dir, FLAGS_tablet_id, FLAGS_schema_hash); + if (s != OLAP_SUCCESS) { + std::cout << "delete tablet meta failed for tablet_id:" << FLAGS_tablet_id + << ", schema_hash:" << FLAGS_schema_hash + << ", status:" << s << std::endl; + return; } - Status st = store->init(); - if (!st.ok()) { - std::cout << "store load failed" << std::endl; - return -1; + std::cout << "delete meta successfully" << std::endl; +} + +int main(int argc, char **argv) { + std::string usage = get_usage(argv[0]); + std::cout << "usage:" << usage << std::endl; + gflags::SetUsageMessage(usage); + google::ParseCommandLineFlags(&argc, &argv, true); + std::cout << "FLAGS_help:" << FLAGS_help << std::endl; + + if (FLAGS_operation == "show_meta") { + show_meta(); } + else { + // operations that need root path should be written here + std::set valid_operations = { + "get_meta", + "load_meta", + "delete_meta" + }; + if (valid_operations.find(FLAGS_operation) == valid_operations.end()) { + std::cout << "invalid operation:" << FLAGS_operation << std::endl; + return -1; + } - if (FLAGS_operation == "get_header") { - std::string value; - OLAPStatus s = TabletMetaManager::get_json_header(store.get(), FLAGS_tablet_id, FLAGS_schema_hash, &value); - if (s == doris::OLAP_ERR_META_KEY_NOT_FOUND) { - std::cout << "no header for tablet_id:" << FLAGS_tablet_id - << " schema_hash:" << FLAGS_schema_hash; - return 0; + path root_path(FLAGS_root_path); + try { + root_path = canonical(root_path); } - std::cout << value << std::endl; - } else if (FLAGS_operation == "flag") { - bool converted = false; - OLAPStatus s = TabletMetaManager::get_header_converted(store.get(), converted); - if (s != OLAP_SUCCESS) { - std::cout << "get header converted flag failed" << std::endl; + catch (...) { + std::cout << "invalid root path:" << FLAGS_root_path << std::endl; return -1; } - std::cout << "is_header_converted is " << converted << std::endl; - } else if (FLAGS_operation == "load_header") { - OLAPStatus s = TabletMetaManager::load_json_header(store.get(), FLAGS_json_header_path); - if (s != OLAP_SUCCESS) { - std::cout << "load header failed" << std::endl; + + std::unique_ptr data_dir(new (std::nothrow) DataDir(root_path.string())); + if (data_dir == nullptr) { + std::cout << "new data dir failed" << std::endl; return -1; } - std::cout << "load header successfully" << std::endl; - } else if (FLAGS_operation == "delete_header") { - OLAPStatus s = TabletMetaManager::remove(store.get(), FLAGS_tablet_id, FLAGS_schema_hash); - if (s != OLAP_SUCCESS) { - std::cout << "delete header failed for tablet_id:" << FLAGS_tablet_id - << " schema_hash:" << FLAGS_schema_hash << std::endl; + Status st = data_dir->init(); + if (!st.ok()) { + std::cout << "data_dir load failed" << std::endl; return -1; } - std::cout << "delete header successfully" << std::endl; - } else if (FLAGS_operation == "rollback") { - auto rollback_func = [&root_path](long tablet_id, - long schema_hash, const std::string& value) -> bool { - TabletMeta tablet_meta; - bool parsed = tablet_meta.deserialize(value); - if (!parsed) { - std::cout << "parse header failed"; - return true; - } - std::string tablet_id_str = std::to_string(tablet_id); - std::string schema_hash_path = root_path + "/data/" + std::to_string(tablet_meta.shard_id()) - + "/" + tablet_id_str + "/" + std::to_string(schema_hash); - std::string header_file_path = schema_hash_path + "/" + tablet_id_str + ".hdr"; - std::cout << "save header to path:" << header_file_path << std::endl; - OLAPStatus s = tablet_meta.save(header_file_path); - if (s != OLAP_SUCCESS) { - std::cout << "save header file to path:" << header_file_path << " failed" << std::endl; - } - return true; - }; - TabletMetaManager::traverse_headers(store->get_meta(), rollback_func); - } else if (FLAGS_operation == "show_header") { - TabletMeta header(FLAGS_pb_header_path); - OLAPStatus s = header.load_and_init(); - if (s != OLAP_SUCCESS) { - std::cout << "load pb header file:" << FLAGS_pb_header_path << " failed" << std::endl; + + if (FLAGS_operation == "get_meta") { + get_meta(data_dir.get()); + } else if (FLAGS_operation == "load_meta") { + load_meta(data_dir.get()); + } else if (FLAGS_operation == "delete_meta") { + delete_meta(data_dir.get()); + } else { + std::cout << "shit usage:" << usage << std::endl; + std::cout << "invalid operation:" << FLAGS_operation << "\n" + << usage << std::endl; + std::cout << "finished print usage" << std::endl; return -1; } - std::string json_header; - json2pb::Pb2JsonOptions json_options; - json_options.pretty_json = true; - TabletMetaPB tablet_meta_pb; - header.to_tablet_pb(&tablet_meta_pb); - json2pb::ProtoMessageToJson(tablet_meta_pb, &json_header, json_options); - std::cout << "header:" << std::endl; - std::cout << json_header << std::endl; - } else { - std::cout << "invalid operation:" << FLAGS_operation << std::endl; - print_usage(argv[0]); - return -1; } + gflags::ShutDownCommandLineFlags(); return 0; } From 59c2a0a989338f7d4fbf491c17d3418c773f1a6a Mon Sep 17 00:00:00 2001 From: huangkangping Date: Wed, 19 Jun 2019 19:30:15 +0800 Subject: [PATCH 2/3] remove addtional log --- be/src/olap/tablet_meta_manager.cpp | 6 ++---- be/src/tools/meta_tool.cpp | 7 +------ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp index f6cbddbc389919..7a6cdce7cd2e8d 100755 --- a/be/src/olap/tablet_meta_manager.cpp +++ b/be/src/olap/tablet_meta_manager.cpp @@ -145,14 +145,12 @@ OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta, OLAPStatus TabletMetaManager::load_json_header(DataDir* store, const std::string& header_path) { std::ifstream infile(header_path); - char buffer[100000]; + char buffer[102400]; std::string json_header; - std::cout << "start to read json file" << std::endl; while (!infile.eof()) { - infile.getline(buffer, 100000); + infile.getline(buffer, 102400); json_header = json_header + buffer; } - std::cout << "json_header:" << json_header << std::endl; boost::algorithm::trim(json_header); TabletMetaPB tablet_meta_pb; bool ret = json2pb::JsonToProtoMessage(json_header, &tablet_meta_pb); diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index ed851d6d210e9f..cac5e6c540b024 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -43,13 +43,12 @@ using doris::TabletMetaManager; const std::string HEADER_PREFIX = "tabletmeta_"; -DECLARE_bool(help); DEFINE_string(root_path, "", "storage root path"); DEFINE_string(operation, "get_meta", "valid operation: get_meta, flag, load_meta, delete_meta, show_meta"); DEFINE_int64(tablet_id, 0, "tablet_id for tablet meta"); DEFINE_int32(schema_hash, 0, "schema_hash for tablet meta"); -DEFINE_string(json_meta_path, "", "json meta file path"); +DEFINE_string(json_meta_path, "", "absolute json meta file path"); DEFINE_string(pb_meta_path, "", "pb meta file path"); std::string get_usage(std::string progname) { @@ -116,10 +115,8 @@ void delete_meta(DataDir *data_dir) { int main(int argc, char **argv) { std::string usage = get_usage(argv[0]); - std::cout << "usage:" << usage << std::endl; gflags::SetUsageMessage(usage); google::ParseCommandLineFlags(&argc, &argv, true); - std::cout << "FLAGS_help:" << FLAGS_help << std::endl; if (FLAGS_operation == "show_meta") { show_meta(); @@ -163,10 +160,8 @@ int main(int argc, char **argv) { } else if (FLAGS_operation == "delete_meta") { delete_meta(data_dir.get()); } else { - std::cout << "shit usage:" << usage << std::endl; std::cout << "invalid operation:" << FLAGS_operation << "\n" << usage << std::endl; - std::cout << "finished print usage" << std::endl; return -1; } } From 9eb0b5656d299071d3bc0fa1f024e626a9d3b578 Mon Sep 17 00:00:00 2001 From: huangkangping Date: Thu, 20 Jun 2019 10:24:36 +0800 Subject: [PATCH 3/3] fix pr problems --- be/src/http/action/meta_action.cpp | 10 ++++---- be/src/olap/snapshot_manager.cpp | 6 ++--- be/src/olap/tablet.cpp | 2 +- be/src/olap/tablet_manager.cpp | 2 +- be/src/olap/tablet_meta_manager.cpp | 25 +++++++++---------- be/src/olap/tablet_meta_manager.h | 8 +++--- .../task/engine_storage_migration_task.cpp | 4 +-- be/src/tools/meta_tool.cpp | 7 +++--- be/test/olap/tablet_meta_manager_test.cpp | 22 ++++++++-------- be/test/olap/tablet_mgr_test.cpp | 2 +- 10 files changed, 43 insertions(+), 45 deletions(-) diff --git a/be/src/http/action/meta_action.cpp b/be/src/http/action/meta_action.cpp index ad01316b6c957b..40d03ac13cf788 100644 --- a/be/src/http/action/meta_action.cpp +++ b/be/src/http/action/meta_action.cpp @@ -38,7 +38,7 @@ namespace doris { const static std::string HEADER_JSON = "application/json"; -Status MetaAction::_handle_header(HttpRequest *req, std::string* json_header) { +Status MetaAction::_handle_header(HttpRequest *req, std::string* json_meta) { req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str()); std::string req_tablet_id = req->param(TABLET_ID_KEY); std::string req_schema_hash = req->param(TABLET_SCHEMA_HASH_KEY); @@ -54,7 +54,7 @@ Status MetaAction::_handle_header(HttpRequest *req, std::string* json_header) { LOG(WARNING) << "no tablet for tablet_id:" << tablet_id << " schema hash:" << schema_hash; return Status("no tablet exist"); } - OLAPStatus s = TabletMetaManager::get_json_header(tablet->data_dir(), tablet_id, schema_hash, json_header); + OLAPStatus s = TabletMetaManager::get_json_meta(tablet->data_dir(), tablet_id, schema_hash, json_meta); if (s == OLAP_ERR_META_KEY_NOT_FOUND) { return Status("no header exist"); } else if (s != OLAP_SUCCESS) { @@ -65,12 +65,12 @@ Status MetaAction::_handle_header(HttpRequest *req, std::string* json_header) { void MetaAction::handle(HttpRequest *req) { if (_meta_type == META_TYPE::HEADER) { - std::string json_header; - Status status = _handle_header(req, &json_header); + std::string json_meta; + Status status = _handle_header(req, &json_meta); std::string status_result = to_json(status); LOG(INFO) << "handle request result:" << status_result; if (status.ok()) { - HttpChannel::send_reply(req, HttpStatus::OK, json_header); + HttpChannel::send_reply(req, HttpStatus::OK, json_meta); } else { HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result); } diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index db7bd31787bfea..0a59184472f428 100755 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -399,7 +399,7 @@ OLAPStatus SnapshotManager::_create_snapshot_files( if (res != OLAP_SUCCESS) { break; } - res = TabletMetaManager::get_header(data_dir, ref_tablet->tablet_id(), + res = TabletMetaManager::get_meta(data_dir, ref_tablet->tablet_id(), ref_tablet->schema_hash(), new_tablet_meta); if (res != OLAP_SUCCESS) { LOG(WARNING) << "fail to load header. res=" << res @@ -443,7 +443,7 @@ OLAPStatus SnapshotManager::_create_snapshot_files( break; } - res = TabletMetaManager::get_header(data_dir, ref_tablet->tablet_id(), + res = TabletMetaManager::get_meta(data_dir, ref_tablet->tablet_id(), ref_tablet->schema_hash(), new_tablet_meta); if (res != OLAP_SUCCESS) { LOG(WARNING) << "fail to load header. res=" << res @@ -547,7 +547,7 @@ OLAPStatus SnapshotManager::_append_single_delta( return OLAP_ERR_MALLOC_ERROR; } - res = TabletMetaManager::get_header(store, request.tablet_id, request.schema_hash, new_tablet_meta); + res = TabletMetaManager::get_meta(store, request.tablet_id, request.schema_hash, new_tablet_meta); if (res != OLAP_SUCCESS) { LOG(WARNING) << "fail to create tablet from header file. " << " tablet_id=" << request.tablet_id diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 857c686b71170e..bb004fe17f75af 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -190,7 +190,7 @@ OLAPStatus Tablet::revise_tablet_meta( do { // load new local tablet_meta to operate on TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta()); - RETURN_NOT_OK(TabletMetaManager::get_header(_data_dir, tablet_id(), schema_hash(), new_tablet_meta)); + RETURN_NOT_OK(TabletMetaManager::get_meta(_data_dir, tablet_id(), schema_hash(), new_tablet_meta)); // delete versions from new local tablet_meta for (const Version& version : versions_to_delete) { diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 51f613e11f17c6..aa374f5ebf9fea 100755 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -1015,7 +1015,7 @@ OLAPStatus TabletManager::start_trash_sweep() { ++it; continue; } - OLAPStatus check_st = TabletMetaManager::get_header((*it)->data_dir(), + OLAPStatus check_st = TabletMetaManager::get_meta((*it)->data_dir(), (*it)->tablet_id(), (*it)->schema_hash(), new_tablet_meta); if (check_st == OLAP_SUCCESS) { if (new_tablet_meta->tablet_state() != TABLET_SHUTDOWN diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp index 7a6cdce7cd2e8d..f9f569db6887ef 100755 --- a/be/src/olap/tablet_meta_manager.cpp +++ b/be/src/olap/tablet_meta_manager.cpp @@ -44,7 +44,7 @@ using rocksdb::kDefaultColumnFamilyName; namespace doris { -OLAPStatus TabletMetaManager::get_header( +OLAPStatus TabletMetaManager::get_meta( DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, TabletMetaSharedPtr tablet_meta) { @@ -64,16 +64,16 @@ OLAPStatus TabletMetaManager::get_header( return tablet_meta->deserialize(value); } -OLAPStatus TabletMetaManager::get_json_header(DataDir* store, - TTabletId tablet_id, TSchemaHash schema_hash, std::string* json_header) { +OLAPStatus TabletMetaManager::get_json_meta(DataDir* store, + TTabletId tablet_id, TSchemaHash schema_hash, std::string* json_meta) { TabletMetaSharedPtr tablet_meta(new TabletMeta()); - OLAPStatus s = get_header(store, tablet_id, schema_hash, tablet_meta); + OLAPStatus s = get_meta(store, tablet_id, schema_hash, tablet_meta); if (s != OLAP_SUCCESS) { return s; } json2pb::Pb2JsonOptions json_options; json_options.pretty_json = true; - tablet_meta->to_json(json_header, json_options); + tablet_meta->to_json(json_meta, json_options); return OLAP_SUCCESS; } @@ -143,17 +143,17 @@ OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta, return status; } -OLAPStatus TabletMetaManager::load_json_header(DataDir* store, const std::string& header_path) { - std::ifstream infile(header_path); +OLAPStatus TabletMetaManager::load_json_meta(DataDir* store, const std::string& meta_path) { + std::ifstream infile(meta_path); char buffer[102400]; - std::string json_header; + std::string json_meta; while (!infile.eof()) { infile.getline(buffer, 102400); - json_header = json_header + buffer; + json_meta = json_meta + buffer; } - boost::algorithm::trim(json_header); + boost::algorithm::trim(json_meta); TabletMetaPB tablet_meta_pb; - bool ret = json2pb::JsonToProtoMessage(json_header, &tablet_meta_pb); + bool ret = json2pb::JsonToProtoMessage(json_meta, &tablet_meta_pb); if (!ret) { return OLAP_ERR_HEADER_LOAD_JSON_HEADER; } @@ -162,14 +162,13 @@ OLAPStatus TabletMetaManager::load_json_header(DataDir* store, const std::string tablet_meta_pb.SerializeToString(&meta_binary); TTabletId tablet_id = tablet_meta_pb.tablet_id(); TSchemaHash schema_hash = tablet_meta_pb.schema_hash(); - std::cout << "start to save meta" << std::endl; return save(store, tablet_id, schema_hash, meta_binary); } OLAPStatus TabletMetaManager::dump_header(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, const std::string& dump_path) { TabletMetaSharedPtr tablet_meta(new TabletMeta()); - OLAPStatus res = TabletMetaManager::get_header(store, tablet_id, schema_hash, tablet_meta); + OLAPStatus res = TabletMetaManager::get_meta(store, tablet_id, schema_hash, tablet_meta); if (res != OLAP_SUCCESS) { return res; } diff --git a/be/src/olap/tablet_meta_manager.h b/be/src/olap/tablet_meta_manager.h index 050a570df0feb7..3898961914e3cd 100644 --- a/be/src/olap/tablet_meta_manager.h +++ b/be/src/olap/tablet_meta_manager.h @@ -33,11 +33,11 @@ const std::string HEADER_PREFIX = "tabletmeta_"; // Helper Class for managing tablet headers of one root path. class TabletMetaManager { public: - static OLAPStatus get_header(DataDir* store, TTabletId tablet_id, + static OLAPStatus get_meta(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, TabletMetaSharedPtr tablet_meta); - static OLAPStatus get_json_header(DataDir* store, TTabletId tablet_id, - TSchemaHash schema_hash, std::string* json_header); + static OLAPStatus get_json_meta(DataDir* store, TTabletId tablet_id, + TSchemaHash schema_hash, std::string* json_meta); static OLAPStatus save(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, TabletMetaSharedPtr tablet_meta, const string& header_prefix = "tabletmeta_"); @@ -50,7 +50,7 @@ class TabletMetaManager { static OLAPStatus traverse_headers(OlapMeta* meta, std::function const& func, const string& header_prefix = "tabletmeta_"); - static OLAPStatus load_json_header(DataDir* store, const std::string& header_path); + static OLAPStatus load_json_meta(DataDir* store, const std::string& meta_path); static OLAPStatus dump_header(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash, const std::string& path); diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index fb1f2b944274a8..1da27abe6f57b1 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -134,7 +134,7 @@ OLAPStatus EngineStorageMigrationTask::_storage_medium_migrate( remove_all_dir(schema_hash_path); } TabletMetaSharedPtr new_tablet_meta(new(std::nothrow) TabletMeta()); - res = TabletMetaManager::get_header(stores[0], tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta); + res = TabletMetaManager::get_meta(stores[0], tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta); if (res != OLAP_ERR_META_KEY_NOT_FOUND) { LOG(WARNING) << "tablet_meta already exists. " << "data_dir:" << stores[0]->path() @@ -218,7 +218,7 @@ OLAPStatus EngineStorageMigrationTask::_generate_new_header( return OLAP_ERR_HEADER_INIT_FAILED; } OLAPStatus res = OLAP_SUCCESS; - TabletMetaManager::get_header(tablet->data_dir(), tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta); + TabletMetaManager::get_meta(tablet->data_dir(), tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta); vector rs_metas; for (auto& rs : consistent_rowsets) { diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index cac5e6c540b024..d6d9476130eb03 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -51,7 +51,7 @@ DEFINE_int32(schema_hash, 0, "schema_hash for tablet meta"); DEFINE_string(json_meta_path, "", "absolute json meta file path"); DEFINE_string(pb_meta_path, "", "pb meta file path"); -std::string get_usage(std::string progname) { +std::string get_usage(const std::string& progname) { std::stringstream ss; ss << progname << " is the Doris BE Meta tool.\n"; ss << "Stop BE first before use this tool.\n"; @@ -77,13 +77,12 @@ void show_meta() { doris::TabletMetaPB tablet_meta_pb; tablet_meta.to_meta_pb(&tablet_meta_pb); json2pb::ProtoMessageToJson(tablet_meta_pb, &json_meta, json_options); - std::cout << "tablet meta:" << std::endl; std::cout << json_meta << std::endl; } void get_meta(DataDir *data_dir) { std::string value; - OLAPStatus s = TabletMetaManager::get_json_header(data_dir, FLAGS_tablet_id, FLAGS_schema_hash, &value); + OLAPStatus s = TabletMetaManager::get_json_meta(data_dir, FLAGS_tablet_id, FLAGS_schema_hash, &value); if (s == doris::OLAP_ERR_META_KEY_NOT_FOUND) { std::cout << "no tablet meta for tablet_id:" << FLAGS_tablet_id << ", schema_hash:" << FLAGS_schema_hash << std::endl; @@ -94,7 +93,7 @@ void get_meta(DataDir *data_dir) { void load_meta(DataDir *data_dir) { // load json tablet meta into meta - OLAPStatus s = TabletMetaManager::load_json_header(data_dir, FLAGS_json_meta_path); + OLAPStatus s = TabletMetaManager::load_json_meta(data_dir, FLAGS_json_meta_path); if (s != OLAP_SUCCESS) { std::cout << "load meta failed, status:" << s << std::endl; return; diff --git a/be/test/olap/tablet_meta_manager_test.cpp b/be/test/olap/tablet_meta_manager_test.cpp index 1e49f73f0436a9..5a39d73066dee9 100755 --- a/be/test/olap/tablet_meta_manager_test.cpp +++ b/be/test/olap/tablet_meta_manager_test.cpp @@ -35,7 +35,7 @@ using std::string; namespace doris { -const std::string header_path = "./be/test/olap/test_data/header.txt"; +const std::string meta_path = "./be/test/olap/test_data/header.txt"; class TabletMetaManagerTest : public testing::Test { public: @@ -48,7 +48,7 @@ class TabletMetaManagerTest : public testing::Test { ASSERT_TRUE(st.ok()); ASSERT_TRUE(boost::filesystem::exists(root_path + "/meta")); - std::ifstream infile(header_path); + std::ifstream infile(meta_path); char buffer[1024]; while (!infile.eof()) { infile.getline(buffer, 1024); @@ -83,26 +83,26 @@ TEST_F(TabletMetaManagerTest, TestSaveAndGetAndRemove) { s = TabletMetaManager::save(_data_dir, tablet_id, schema_hash, tablet_meta); ASSERT_EQ(OLAP_SUCCESS, s); - std::string json_header_read; - s = TabletMetaManager::get_json_header(_data_dir, tablet_id, schema_hash, &json_header_read); + std::string json_meta_read; + s = TabletMetaManager::get_json_meta(_data_dir, tablet_id, schema_hash, &json_meta_read); ASSERT_EQ(OLAP_SUCCESS, s); - ASSERT_EQ(_json_header, json_header_read); + ASSERT_EQ(_json_header, json_meta_read); s = TabletMetaManager::remove(_data_dir, tablet_id, schema_hash); ASSERT_EQ(OLAP_SUCCESS, s); - TabletMetaSharedPtr header_read(new TabletMeta()); - s = TabletMetaManager::get_header(_data_dir, tablet_id, schema_hash, header_read); + TabletMetaSharedPtr meta_read(new TabletMeta()); + s = TabletMetaManager::get_meta(_data_dir, tablet_id, schema_hash, meta_read); ASSERT_EQ(OLAP_ERR_META_KEY_NOT_FOUND, s); } TEST_F(TabletMetaManagerTest, TestLoad) { const TTabletId tablet_id = 15672; const TSchemaHash schema_hash = 567997577; - OLAPStatus s = TabletMetaManager::load_json_header(_data_dir, header_path); + OLAPStatus s = TabletMetaManager::load_json_meta(_data_dir, meta_path); ASSERT_EQ(OLAP_SUCCESS, s); - std::string json_header_read; - s = TabletMetaManager::get_json_header(_data_dir, tablet_id, schema_hash, &json_header_read); + std::string json_meta_read; + s = TabletMetaManager::get_json_meta(_data_dir, tablet_id, schema_hash, &json_meta_read); ASSERT_EQ(OLAP_SUCCESS, s); - ASSERT_EQ(_json_header, json_header_read); + ASSERT_EQ(_json_header, json_meta_read); } } // namespace doris diff --git a/be/test/olap/tablet_mgr_test.cpp b/be/test/olap/tablet_mgr_test.cpp index c1b0b2853cc57d..498fa25cd9bfce 100644 --- a/be/test/olap/tablet_mgr_test.cpp +++ b/be/test/olap/tablet_mgr_test.cpp @@ -132,7 +132,7 @@ TEST_F(TabletMgrTest, CreateTablet) { ASSERT_TRUE(dir_exist); // check meta has this tablet TabletMetaSharedPtr new_tablet_meta(new TabletMeta()); - OLAPStatus check_meta_st = TabletMetaManager::get_header(_data_dir, 111, 3333, new_tablet_meta); + OLAPStatus check_meta_st = TabletMetaManager::get_meta(_data_dir, 111, 3333, new_tablet_meta); ASSERT_TRUE(check_meta_st == OLAP_SUCCESS); // retry create should be successfully