From 3eb471e060889241f737c18c1d3cfd5077c75568 Mon Sep 17 00:00:00 2001 From: cyongli Date: Wed, 14 Nov 2018 16:57:58 +0800 Subject: [PATCH 1/3] Transform row-oriented table to columnar-oriented table --- be/src/olap/olap_header_manager.cpp | 14 +------------- be/src/olap/olap_header_manager.h | 2 +- be/src/olap/store.cpp | 16 ++++++++++++---- be/src/tools/meta_tool.cpp | 13 +++++++++++-- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/be/src/olap/olap_header_manager.cpp b/be/src/olap/olap_header_manager.cpp index 9bd04dca654773..a6fccd13433c66 100755 --- a/be/src/olap/olap_header_manager.cpp +++ b/be/src/olap/olap_header_manager.cpp @@ -126,19 +126,7 @@ OLAPStatus OlapHeaderManager::set_converted_flag(OlapStore* store) { } OLAPStatus OlapHeaderManager::traverse_headers(OlapMeta* meta, - std::function const& func) { - auto traverse_header_func = [&func](const std::string& key, const std::string& value) -> bool { - std::vector parts; - // key format: "hdr_" + tablet_id + "_" + schema_hash - split_string(key, '_', &parts); - if (parts.size() != 3) { - LOG(WARNING) << "invalid header key:" << key << ", splitted size:" << parts.size(); - return true; - } - TTabletId tablet_id = std::stol(parts[1].c_str(), NULL, 10); - TSchemaHash schema_hash = std::stol(parts[2].c_str(), NULL, 10); - return func(tablet_id, schema_hash, value); - }; + std::function const& traverse_header_func) { OLAPStatus status = meta->iterate(META_COLUMN_FAMILY_INDEX, HEADER_PREFIX, traverse_header_func); return status; } diff --git a/be/src/olap/olap_header_manager.h b/be/src/olap/olap_header_manager.h index c74be38c3fe485..ba5d383924b6b5 100644 --- a/be/src/olap/olap_header_manager.h +++ b/be/src/olap/olap_header_manager.h @@ -39,7 +39,7 @@ class OlapHeaderManager { static OLAPStatus remove(OlapStore* store, TTabletId tablet_id, TSchemaHash schema_hash); static OLAPStatus traverse_headers(OlapMeta* meta, - std::function const& func); + std::function const& traverse_header_func); static OLAPStatus get_header_converted(OlapStore* store, bool& flag); diff --git a/be/src/olap/store.cpp b/be/src/olap/store.cpp index 5842708d1ba141..dfc334a88af77e 100755 --- a/be/src/olap/store.cpp +++ b/be/src/olap/store.cpp @@ -543,16 +543,24 @@ OLAPStatus OlapStore::_load_table_from_header(OLAPEngine* engine, TTabletId tabl } OLAPStatus OlapStore::load_tables(OLAPEngine* engine) { - auto load_table_func = [this, engine](long tablet_id, - long schema_hash, const std::string& value) -> bool { + auto traverse_header_func = [this, engine](const std::string& key, const std::string& value) -> bool { + std::vector parts; + // key format: "hdr_" + tablet_id + "_" + schema_hash + split_string(key, '_', &parts); + if (parts.size() != 3) { + LOG(WARNING) << "invalid header key:" << key << ", splitted size:" << parts.size(); + return true; + } + TTabletId tablet_id = std::stol(parts[1].c_str(), NULL, 10); + TSchemaHash schema_hash = std::stol(parts[2].c_str(), NULL, 10); OLAPStatus status = _load_table_from_header(engine, tablet_id, schema_hash, value); if (status != OLAP_SUCCESS) { LOG(WARNING) << "load table from header failed. status:" << status - << "tablet=" << tablet_id << "." << schema_hash; + << "tablet=" << tablet_id << "." << schema_hash; }; return true; }; - OLAPStatus status = OlapHeaderManager::traverse_headers(_meta, load_table_func); + OLAPStatus status = OlapHeaderManager::traverse_headers(_meta, traverse_header_func); return status; } diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index 39b51ca9abec07..0901c514db50ba 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -127,8 +127,17 @@ int main(int argc, char** argv) { } std::cout << "delete header successfully" << std::endl; } else if (FLAGS_operation == "rollback") { - auto rollback_func = [&root_path](long tablet_id, - long schema_hash, const std::string& value) -> bool { + auto rollback_func = [&root_path](const std::string& key, + const std::string& value) -> bool { + std::vector parts; + // key format: "hdr_" + tablet_id + "_" + schema_hash + doris::split_string(key, '_', &parts); + if (parts.size() != 3) { + LOG(WARNING) << "invalid header key:" << key << ", splitted size:" << parts.size(); + return true; + } + int64_t tablet_id = std::stol(parts[1].c_str(), NULL, 10); + int64_t schema_hash = std::stol(parts[2].c_str(), NULL, 10); OLAPHeader olap_header; bool parsed = olap_header.ParseFromString(value); if (!parsed) { From 3c48fab9f37388623515407988ef8f028d5bc4ba Mon Sep 17 00:00:00 2001 From: cyongli Date: Fri, 16 Nov 2018 15:01:30 +0800 Subject: [PATCH 2/3] Transform row-oriented table to columnar-oriented table --- be/src/olap/olap_header_manager.cpp | 14 +++++++++++++- be/src/olap/olap_header_manager.h | 2 +- be/src/olap/store.cpp | 16 ++++------------ be/src/tools/meta_tool.cpp | 13 ++----------- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/be/src/olap/olap_header_manager.cpp b/be/src/olap/olap_header_manager.cpp index a6fccd13433c66..9bd04dca654773 100755 --- a/be/src/olap/olap_header_manager.cpp +++ b/be/src/olap/olap_header_manager.cpp @@ -126,7 +126,19 @@ OLAPStatus OlapHeaderManager::set_converted_flag(OlapStore* store) { } OLAPStatus OlapHeaderManager::traverse_headers(OlapMeta* meta, - std::function const& traverse_header_func) { + std::function const& func) { + auto traverse_header_func = [&func](const std::string& key, const std::string& value) -> bool { + std::vector parts; + // key format: "hdr_" + tablet_id + "_" + schema_hash + split_string(key, '_', &parts); + if (parts.size() != 3) { + LOG(WARNING) << "invalid header key:" << key << ", splitted size:" << parts.size(); + return true; + } + TTabletId tablet_id = std::stol(parts[1].c_str(), NULL, 10); + TSchemaHash schema_hash = std::stol(parts[2].c_str(), NULL, 10); + return func(tablet_id, schema_hash, value); + }; OLAPStatus status = meta->iterate(META_COLUMN_FAMILY_INDEX, HEADER_PREFIX, traverse_header_func); return status; } diff --git a/be/src/olap/olap_header_manager.h b/be/src/olap/olap_header_manager.h index ba5d383924b6b5..c74be38c3fe485 100644 --- a/be/src/olap/olap_header_manager.h +++ b/be/src/olap/olap_header_manager.h @@ -39,7 +39,7 @@ class OlapHeaderManager { static OLAPStatus remove(OlapStore* store, TTabletId tablet_id, TSchemaHash schema_hash); static OLAPStatus traverse_headers(OlapMeta* meta, - std::function const& traverse_header_func); + std::function const& func); static OLAPStatus get_header_converted(OlapStore* store, bool& flag); diff --git a/be/src/olap/store.cpp b/be/src/olap/store.cpp index dfc334a88af77e..5842708d1ba141 100755 --- a/be/src/olap/store.cpp +++ b/be/src/olap/store.cpp @@ -543,24 +543,16 @@ OLAPStatus OlapStore::_load_table_from_header(OLAPEngine* engine, TTabletId tabl } OLAPStatus OlapStore::load_tables(OLAPEngine* engine) { - auto traverse_header_func = [this, engine](const std::string& key, const std::string& value) -> bool { - std::vector parts; - // key format: "hdr_" + tablet_id + "_" + schema_hash - split_string(key, '_', &parts); - if (parts.size() != 3) { - LOG(WARNING) << "invalid header key:" << key << ", splitted size:" << parts.size(); - return true; - } - TTabletId tablet_id = std::stol(parts[1].c_str(), NULL, 10); - TSchemaHash schema_hash = std::stol(parts[2].c_str(), NULL, 10); + auto load_table_func = [this, engine](long tablet_id, + long schema_hash, const std::string& value) -> bool { OLAPStatus status = _load_table_from_header(engine, tablet_id, schema_hash, value); if (status != OLAP_SUCCESS) { LOG(WARNING) << "load table from header failed. status:" << status - << "tablet=" << tablet_id << "." << schema_hash; + << "tablet=" << tablet_id << "." << schema_hash; }; return true; }; - OLAPStatus status = OlapHeaderManager::traverse_headers(_meta, traverse_header_func); + OLAPStatus status = OlapHeaderManager::traverse_headers(_meta, load_table_func); return status; } diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index 0901c514db50ba..39b51ca9abec07 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -127,17 +127,8 @@ int main(int argc, char** argv) { } std::cout << "delete header successfully" << std::endl; } else if (FLAGS_operation == "rollback") { - auto rollback_func = [&root_path](const std::string& key, - const std::string& value) -> bool { - std::vector parts; - // key format: "hdr_" + tablet_id + "_" + schema_hash - doris::split_string(key, '_', &parts); - if (parts.size() != 3) { - LOG(WARNING) << "invalid header key:" << key << ", splitted size:" << parts.size(); - return true; - } - int64_t tablet_id = std::stol(parts[1].c_str(), NULL, 10); - int64_t schema_hash = std::stol(parts[2].c_str(), NULL, 10); + auto rollback_func = [&root_path](long tablet_id, + long schema_hash, const std::string& value) -> bool { OLAPHeader olap_header; bool parsed = olap_header.ParseFromString(value); if (!parsed) { From 071c81047dfe6ef414cd0e584bdb44cc3ae04d27 Mon Sep 17 00:00:00 2001 From: cyongli Date: Fri, 16 Nov 2018 16:36:31 +0800 Subject: [PATCH 3/3] change log verbose level --- be/src/exec/olap_scan_node.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp index cc1897e9ca8808..b0a456d99b934e 100644 --- a/be/src/exec/olap_scan_node.cpp +++ b/be/src/exec/olap_scan_node.cpp @@ -702,8 +702,8 @@ Status OlapScanNode::normalize_in_predicate(SlotDescriptor* slot, ColumnValueRan // 1.2 Skip if InPredicate value size larger then max_scan_key_num if (pred->hybird_set()->size() > config::doris_max_scan_key_num) { - LOG(WARNING) << "Predicate value num " << pred->hybird_set()->size() - << " excede limit " << config::doris_max_scan_key_num; + VLOG(3) << "Predicate value num " << pred->hybird_set()->size() + << " excede limit " << config::doris_max_scan_key_num; continue; }