Skip to content

Be refactor optimize meta tool new #1337

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chaoyli merged 3 commits into from
Jun 20, 2019
Merged

Be refactor optimize meta tool new #1337

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3832eea
add meta tool
kangpinghuang Jun 18, 2019
59c2a0a
remove addtional log
kangpinghuang Jun 19, 2019
9eb0b56
fix pr problems
kangpinghuang Jun 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion be/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,7 @@ add_subdirectory(${SRC_DIR}/exprs)
add_subdirectory(${SRC_DIR}/udf)
add_subdirectory(${SRC_DIR}/runtime)
add_subdirectory(${SRC_DIR}/testutil)
#add_subdirectory(${SRC_DIR}/tools)
add_subdirectory(${SRC_DIR}/tools)
add_subdirectory(${SRC_DIR}/udf_samples)
add_subdirectory(${SRC_DIR}/geo)

Expand Down
10 changes: 5 additions & 5 deletions be/src/http/action/meta_action.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ namespace doris {

const static std::string HEADER_JSON = "application/json";

Status MetaAction::_handle_header(HttpRequest *req, std::string* json_header) {
Status MetaAction::_handle_header(HttpRequest *req, std::string* json_meta) {
req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str());
std::string req_tablet_id = req->param(TABLET_ID_KEY);
std::string req_schema_hash = req->param(TABLET_SCHEMA_HASH_KEY);
Expand All @@ -54,7 +54,7 @@ Status MetaAction::_handle_header(HttpRequest *req, std::string* json_header) {
LOG(WARNING) << "no tablet for tablet_id:" << tablet_id << " schema hash:" << schema_hash;
return Status("no tablet exist");
}
OLAPStatus s = TabletMetaManager::get_json_header(tablet->data_dir(), tablet_id, schema_hash, json_header);
OLAPStatus s = TabletMetaManager::get_json_meta(tablet->data_dir(), tablet_id, schema_hash, json_meta);
if (s == OLAP_ERR_META_KEY_NOT_FOUND) {
return Status("no header exist");
} else if (s != OLAP_SUCCESS) {
Expand All @@ -65,12 +65,12 @@ Status MetaAction::_handle_header(HttpRequest *req, std::string* json_header) {

void MetaAction::handle(HttpRequest *req) {
if (_meta_type == META_TYPE::HEADER) {
std::string json_header;
Status status = _handle_header(req, &json_header);
std::string json_meta;
Status status = _handle_header(req, &json_meta);
std::string status_result = to_json(status);
LOG(INFO) << "handle request result:" << status_result;
if (status.ok()) {
HttpChannel::send_reply(req, HttpStatus::OK, json_header);
HttpChannel::send_reply(req, HttpStatus::OK, json_meta);
} else {
HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result);
}
Expand Down
6 changes: 3 additions & 3 deletions be/src/olap/snapshot_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,7 +399,7 @@ OLAPStatus SnapshotManager::_create_snapshot_files(
if (res != OLAP_SUCCESS) {
break;
}
res = TabletMetaManager::get_header(data_dir, ref_tablet->tablet_id(),
res = TabletMetaManager::get_meta(data_dir, ref_tablet->tablet_id(),
ref_tablet->schema_hash(), new_tablet_meta);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "fail to load header. res=" << res
Expand Down Expand Up @@ -443,7 +443,7 @@ OLAPStatus SnapshotManager::_create_snapshot_files(
break;
}

res = TabletMetaManager::get_header(data_dir, ref_tablet->tablet_id(),
res = TabletMetaManager::get_meta(data_dir, ref_tablet->tablet_id(),
ref_tablet->schema_hash(), new_tablet_meta);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "fail to load header. res=" << res
Expand Down Expand Up @@ -547,7 +547,7 @@ OLAPStatus SnapshotManager::_append_single_delta(
return OLAP_ERR_MALLOC_ERROR;
}

res = TabletMetaManager::get_header(store, request.tablet_id, request.schema_hash, new_tablet_meta);
res = TabletMetaManager::get_meta(store, request.tablet_id, request.schema_hash, new_tablet_meta);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "fail to create tablet from header file. "
<< " tablet_id=" << request.tablet_id
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/tablet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ OLAPStatus Tablet::revise_tablet_meta(
do {
// load new local tablet_meta to operate on
TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta());
RETURN_NOT_OK(TabletMetaManager::get_header(_data_dir, tablet_id(), schema_hash(), new_tablet_meta));
RETURN_NOT_OK(TabletMetaManager::get_meta(_data_dir, tablet_id(), schema_hash(), new_tablet_meta));

// delete versions from new local tablet_meta
for (const Version& version : versions_to_delete) {
Expand Down
2 changes: 1 addition & 1 deletion be/src/olap/tablet_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1015,7 +1015,7 @@ OLAPStatus TabletManager::start_trash_sweep() {
++it;
continue;
}
OLAPStatus check_st = TabletMetaManager::get_header((*it)->data_dir(),
OLAPStatus check_st = TabletMetaManager::get_meta((*it)->data_dir(),
(*it)->tablet_id(), (*it)->schema_hash(), new_tablet_meta);
if (check_st == OLAP_SUCCESS) {
if (new_tablet_meta->tablet_state() != TABLET_SHUTDOWN
Expand Down
39 changes: 20 additions & 19 deletions be/src/olap/tablet_meta_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ using rocksdb::kDefaultColumnFamilyName;

namespace doris {

OLAPStatus TabletMetaManager::get_header(
OLAPStatus TabletMetaManager::get_meta(
DataDir* store, TTabletId tablet_id,
TSchemaHash schema_hash,
TabletMetaSharedPtr tablet_meta) {
Expand All @@ -64,16 +64,16 @@ OLAPStatus TabletMetaManager::get_header(
return tablet_meta->deserialize(value);
}

OLAPStatus TabletMetaManager::get_json_header(DataDir* store,
TTabletId tablet_id, TSchemaHash schema_hash, std::string* json_header) {
OLAPStatus TabletMetaManager::get_json_meta(DataDir* store,
TTabletId tablet_id, TSchemaHash schema_hash, std::string* json_meta) {
TabletMetaSharedPtr tablet_meta(new TabletMeta());
OLAPStatus s = get_header(store, tablet_id, schema_hash, tablet_meta);
OLAPStatus s = get_meta(store, tablet_id, schema_hash, tablet_meta);
if (s != OLAP_SUCCESS) {
return s;
}
json2pb::Pb2JsonOptions json_options;
json_options.pretty_json = true;
tablet_meta->to_json(json_header, json_options);
tablet_meta->to_json(json_meta, json_options);
return OLAP_SUCCESS;
}

Expand All @@ -86,9 +86,9 @@ OLAPStatus TabletMetaManager::save(DataDir* store,
std::string value;
tablet_meta->serialize(&value);
OlapMeta* meta = store->get_meta();
LOG(INFO) << "save tablet meta "
<< " tablet_id=" << tablet_id
<< " schema_hash=" << schema_hash;
LOG(INFO) << "save tablet meta"
<< ", key:" << key
<< ", meta length:" << value.length();
return meta->put(META_COLUMN_FAMILY_INDEX, key, value);
}

Expand All @@ -98,6 +98,7 @@ OLAPStatus TabletMetaManager::save(DataDir* store,
key_stream << header_prefix << tablet_id << "_" << schema_hash;
std::string key = key_stream.str();
VLOG(3) << "save tablet meta to meta store: key = " << key;
std::cout << "save tablet meta to meta store: key = " << key << std::endl;
OlapMeta* meta = store->get_meta();

TabletMetaPB de_tablet_meta_pb;
Expand All @@ -107,8 +108,7 @@ OLAPStatus TabletMetaManager::save(DataDir* store,
}

LOG(INFO) << "save tablet meta "
<< " tablet_id=" << tablet_id
<< " schema_hash=" << schema_hash
<< ", key:" << key
<< " meta_size=" << meta_binary.length();
return meta->put(META_COLUMN_FAMILY_INDEX, key, meta_binary);
}
Expand Down Expand Up @@ -143,20 +143,21 @@ OLAPStatus TabletMetaManager::traverse_headers(OlapMeta* meta,
return status;
}

OLAPStatus TabletMetaManager::load_json_header(DataDir* store, const std::string& header_path) {
std::ifstream infile(header_path);
char buffer[1024];
std::string json_header;
OLAPStatus TabletMetaManager::load_json_meta(DataDir* store, const std::string& meta_path) {
std::ifstream infile(meta_path);
char buffer[102400];
std::string json_meta;
while (!infile.eof()) {
infile.getline(buffer, 1024);
json_header = json_header + buffer;
infile.getline(buffer, 102400);
json_meta = json_meta + buffer;
}
boost::algorithm::trim(json_header);
boost::algorithm::trim(json_meta);
TabletMetaPB tablet_meta_pb;
bool ret = json2pb::JsonToProtoMessage(json_header, &tablet_meta_pb);
bool ret = json2pb::JsonToProtoMessage(json_meta, &tablet_meta_pb);
if (!ret) {
return OLAP_ERR_HEADER_LOAD_JSON_HEADER;
}

std::string meta_binary;
tablet_meta_pb.SerializeToString(&meta_binary);
TTabletId tablet_id = tablet_meta_pb.tablet_id();
Expand All @@ -167,7 +168,7 @@ OLAPStatus TabletMetaManager::load_json_header(DataDir* store, const std::string
OLAPStatus TabletMetaManager::dump_header(DataDir* store, TTabletId tablet_id,
TSchemaHash schema_hash, const std::string& dump_path) {
TabletMetaSharedPtr tablet_meta(new TabletMeta());
OLAPStatus res = TabletMetaManager::get_header(store, tablet_id, schema_hash, tablet_meta);
OLAPStatus res = TabletMetaManager::get_meta(store, tablet_id, schema_hash, tablet_meta);
if (res != OLAP_SUCCESS) {
return res;
}
Expand Down
8 changes: 4 additions & 4 deletions be/src/olap/tablet_meta_manager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ const std::string HEADER_PREFIX = "tabletmeta_";
// Helper Class for managing tablet headers of one root path.
class TabletMetaManager {
public:
static OLAPStatus get_header(DataDir* store, TTabletId tablet_id,
static OLAPStatus get_meta(DataDir* store, TTabletId tablet_id,
TSchemaHash schema_hash, TabletMetaSharedPtr tablet_meta);

static OLAPStatus get_json_header(DataDir* store, TTabletId tablet_id,
TSchemaHash schema_hash, std::string* json_header);
static OLAPStatus get_json_meta(DataDir* store, TTabletId tablet_id,
TSchemaHash schema_hash, std::string* json_meta);

static OLAPStatus save(DataDir* store, TTabletId tablet_id, TSchemaHash schema_hash,
TabletMetaSharedPtr tablet_meta, const string& header_prefix = "tabletmeta_");
Expand All @@ -50,7 +50,7 @@ class TabletMetaManager {
static OLAPStatus traverse_headers(OlapMeta* meta,
std::function<bool(long, long, const std::string&)> const& func, const string& header_prefix = "tabletmeta_");

static OLAPStatus load_json_header(DataDir* store, const std::string& header_path);
static OLAPStatus load_json_meta(DataDir* store, const std::string& meta_path);

static OLAPStatus dump_header(DataDir* store, TTabletId tablet_id,
TSchemaHash schema_hash, const std::string& path);
Expand Down
4 changes: 2 additions & 2 deletions be/src/olap/task/engine_storage_migration_task.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ OLAPStatus EngineStorageMigrationTask::_storage_medium_migrate(
remove_all_dir(schema_hash_path);
}
TabletMetaSharedPtr new_tablet_meta(new(std::nothrow) TabletMeta());
res = TabletMetaManager::get_header(stores[0], tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta);
res = TabletMetaManager::get_meta(stores[0], tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta);
if (res != OLAP_ERR_META_KEY_NOT_FOUND) {
LOG(WARNING) << "tablet_meta already exists. "
<< "data_dir:" << stores[0]->path()
Expand Down Expand Up @@ -218,7 +218,7 @@ OLAPStatus EngineStorageMigrationTask::_generate_new_header(
return OLAP_ERR_HEADER_INIT_FAILED;
}
OLAPStatus res = OLAP_SUCCESS;
TabletMetaManager::get_header(tablet->data_dir(), tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta);
TabletMetaManager::get_meta(tablet->data_dir(), tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta);

vector<RowsetMetaSharedPtr> rs_metas;
for (auto& rs : consistent_rowsets) {
Expand Down
Loading