From 70856aac3f6b35901479ae1d93294905c83e08cc Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Mon, 9 Jan 2023 10:15:43 +0800 Subject: [PATCH 01/24] add schema_tables_scanner get_next_block --- be/src/exec/schema_scanner.cpp | 14 + be/src/exec/schema_scanner.h | 7 + .../schema_scanner/schema_tables_scanner.cpp | 305 +++++++++++++++++- .../schema_scanner/schema_tables_scanner.h | 5 + be/src/vec/exec/vschema_scan_node.cpp | 151 +++++---- 5 files changed, 427 insertions(+), 55 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 4f2d029df761a4..436e6b8fd0b7ba 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -34,6 +34,7 @@ #include "exec/schema_scanner/schema_views_scanner.h" #include "runtime/define_primitive_type.h" #include "vec/common/string_ref.h" +#include "vec/core/block.h" namespace doris { @@ -83,6 +84,19 @@ Status SchemaScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { return Status::OK(); } +Status SchemaScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("used before initialized."); + } + + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + *eos = true; + return Status::OK(); +} + Status SchemaScanner::init(SchemaScannerParam* param, ObjectPool* pool) { if (_is_init) { return Status::OK(); diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index 9d2fa1c5fed672..d9c9076796fc3c 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -25,6 +25,7 @@ #include "gen_cpp/Types_types.h" #include "runtime/mem_pool.h" #include "runtime/tuple.h" +#include "vec/core/block.h" namespace doris { @@ -32,6 +33,10 @@ namespace doris { class DorisServer; class RuntimeState; +namespace vectorized { +class Block; +} + // scanner parameter from frontend struct SchemaScannerParam { const std::string* db; @@ -79,10 +84,12 @@ class SchemaScanner { // Start to work virtual Status start(RuntimeState* state); virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + virtual Status get_next_block(vectorized::Block* block, bool* eos); // factory function static SchemaScanner* create(TSchemaTableType::type type); const TupleDescriptor* tuple_desc() const { return _tuple_desc; } + const TSchemaTableType::type type() const { return _schema_table_type; } static void set_doris_server(DorisServer* doris_server) { _s_doris_server = doris_server; } diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index ee9febe3f201db..b52d0ae608d1e9 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -17,8 +17,10 @@ #include "exec/schema_scanner/schema_tables_scanner.h" +#include "common/status.h" #include "exec/schema_scanner/schema_helper.h" #include "runtime/primitive_type.h" +#include "vec/columns/column_complex.h" #include "vec/common/string_ref.h" namespace doris { @@ -52,7 +54,9 @@ SchemaTablesScanner::SchemaTablesScanner() : SchemaScanner(_s_tbls_columns, sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), _db_index(0), - _table_index(0) {} + _table_index(0) { + _schema_table_type = TSchemaTableType::SCH_TABLES; +} SchemaTablesScanner::~SchemaTablesScanner() {} @@ -307,4 +311,303 @@ Status SchemaTablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) return fill_one_row(tuple, pool); } +Status SchemaTablesScanner::fill_dest_column(vectorized::Block* block, void* data, + const SlotDescriptor* slot_desc) { + if (!block->has(slot_desc->col_name())) { + return Status::OK(); + } + vectorized::IColumn* col_ptr = const_cast( + block->get_by_name(slot_desc->col_name()).column.get()); + + if (data == nullptr) { + auto* nullable_column = reinterpret_cast(col_ptr); + nullable_column->get_null_map_data().push_back(1); + nullable_column->insert_data(nullptr, 0); + return Status::OK(); + } + if (slot_desc->is_nullable()) { + auto* nullable_column = reinterpret_cast(col_ptr); + nullable_column->get_null_map_data().push_back(0); + // if (data == nullptr) { + // nullable_column->insert_data(nullptr, 0); + // return Status::OK(); + // } + col_ptr = &nullable_column->get_nested_column(); + } + switch (slot_desc->type().type) { + case TYPE_HLL: { + HyperLogLog* hll_slot = reinterpret_cast(data); + reinterpret_cast(col_ptr)->get_data().emplace_back(*hll_slot); + break; + } + case TYPE_VARCHAR: + case TYPE_CHAR: + case TYPE_STRING: { + StringRef* str_slot = reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data(str_slot->data, + str_slot->size); + break; + } + + case TYPE_BOOLEAN: { + uint8_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_TINYINT: { + int8_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_SMALLINT: { + int16_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_INT: { + int32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_BIGINT: { + int64_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_LARGEINT: { + __int128 num; + memcpy(&num, data, sizeof(__int128)); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_FLOAT: { + float num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_DOUBLE: { + double num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_DATE: { + vectorized::VecDateTimeValue value; + DateTimeValue* ts_slot = reinterpret_cast(data); + value.convert_dt_to_vec_dt(ts_slot); + reinterpret_cast*>(col_ptr)->insert_data( + reinterpret_cast(&value), 0); + break; + } + + case TYPE_DATEV2: { + uint32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_DATETIME: { + vectorized::VecDateTimeValue value; + DateTimeValue* ts_slot = reinterpret_cast(data); + value.convert_dt_to_vec_dt(ts_slot); + reinterpret_cast*>(col_ptr)->insert_data( + reinterpret_cast(&value), 0); + break; + } + + case TYPE_DATETIMEV2: { + uint32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_DECIMALV2: { + const vectorized::Int128 num = (reinterpret_cast(data))->value; + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + case TYPE_DECIMAL128I: { + const vectorized::Int128 num = (reinterpret_cast(data))->value; + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + case TYPE_DECIMAL32: { + const int32_t num = *reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + case TYPE_DECIMAL64: { + const int64_t num = *reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + default: { + DCHECK(false) << "bad slot type: " << slot_desc->type(); + std::stringstream ss; + ss << "Fail to convert schema type:'" << slot_desc->type() << " on column:`" + << slot_desc->col_name() + "`"; + return Status::InternalError(ss.str()); + } + } + + return Status::OK(); +} + +Status SchemaTablesScanner::fill_one_row(vectorized::Block* block) { + const TTableStatus& tbl_status = _table_result.tables[_table_index]; + // catalog + { + std::string catalog_name = _db_result.catalogs[_db_index - 1]; + StringRef str_slot = StringRef(catalog_name.c_str(), catalog_name.size()); + fill_dest_column(block, &str_slot, _tuple_desc->slots()[0]); + } + // schema + { + std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); + StringRef str_slot = StringRef(db_name.c_str(), db_name.size()); + fill_dest_column(block, &str_slot, _tuple_desc->slots()[1]); + } + // name + { + const std::string* src = &tbl_status.name; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _tuple_desc->slots()[2]); + } + // type + { + const std::string* src = &tbl_status.type; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _tuple_desc->slots()[3]); + } + // engine + if (tbl_status.__isset.engine) { + const std::string* src = &tbl_status.engine; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _tuple_desc->slots()[4]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[4]); + } + // version + { fill_dest_column(block, nullptr, _tuple_desc->slots()[5]); } + // row_format + { fill_dest_column(block, nullptr, _tuple_desc->slots()[6]); } + // rows + if (tbl_status.__isset.rows) { + int64_t src = tbl_status.rows; + fill_dest_column(block, &src, _tuple_desc->slots()[7]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[7]); + } + // avg_row_length + if (tbl_status.__isset.avg_row_length) { + int64_t src = tbl_status.avg_row_length; + fill_dest_column(block, &src, _tuple_desc->slots()[8]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[8]); + } + // data_length + if (tbl_status.__isset.avg_row_length) { + int64_t src = tbl_status.data_length; + fill_dest_column(block, &src, _tuple_desc->slots()[9]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[9]); + } // max_data_length + { fill_dest_column(block, nullptr, _tuple_desc->slots()[10]); } + // index_length + { fill_dest_column(block, nullptr, _tuple_desc->slots()[11]); } + // data_free + { fill_dest_column(block, nullptr, _tuple_desc->slots()[12]); } + // auto_increment + { fill_dest_column(block, nullptr, _tuple_desc->slots()[13]); } + // creation_time + if (tbl_status.__isset.create_time) { + int64_t create_time = tbl_status.create_time; + if (create_time <= 0) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[14]); + } else { + DateTimeValue time_slot; + time_slot.from_unixtime(create_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, &time_slot, _tuple_desc->slots()[14]); + } + } + // update_time + if (tbl_status.__isset.update_time) { + int64_t update_time = tbl_status.update_time; + if (update_time <= 0) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[15]); + } else { + DateTimeValue time_slot; + time_slot.from_unixtime(update_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, &time_slot, _tuple_desc->slots()[15]); + } + } + // check_time + if (tbl_status.__isset.last_check_time) { + int64_t check_time = tbl_status.last_check_time; + if (check_time <= 0) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[16]); + } else { + DateTimeValue time_slot; + time_slot.from_unixtime(check_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, &time_slot, _tuple_desc->slots()[16]); + } + } + // collation + if (tbl_status.__isset.collation) { + const std::string* src = &tbl_status.collation; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _tuple_desc->slots()[17]); + + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[17]); + } + // checksum + { fill_dest_column(block, nullptr, _tuple_desc->slots()[18]); } + // create_options + { fill_dest_column(block, nullptr, _tuple_desc->slots()[19]); } + // create_comment + { + const std::string* src = &tbl_status.comment; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _tuple_desc->slots()[20]); + } + _table_index++; + return Status::OK(); +} + +Status SchemaTablesScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + while (_table_index >= _table_result.tables.size()) { + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(get_new_table()); + } else { + *eos = true; + return Status::OK(); + } + } + *eos = false; + return fill_one_row(block); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h index ff957999339eea..3375f72d7b4036 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_tables_scanner.h @@ -17,8 +17,10 @@ #pragma once +#include "common/status.h" #include "exec/schema_scanner.h" #include "gen_cpp/FrontendService_types.h" +#include "vec/core/block.h" namespace doris { @@ -29,10 +31,13 @@ class SchemaTablesScanner : public SchemaScanner { virtual Status start(RuntimeState* state); virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + virtual Status get_next_block(vectorized::Block* block, bool* eos); private: Status get_new_table(); Status fill_one_row(Tuple* tuple, MemPool* pool); + Status fill_one_row(vectorized::Block* block); + Status fill_dest_column(vectorized::Block* block, void* data, const SlotDescriptor* slot_desc); int _db_index; int _table_index; diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index 9d26ddf03b2b06..ee86589c5ba187 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -17,12 +17,16 @@ #include "vec/exec/vschema_scan_node.h" +#include + +#include "common/status.h" #include "exec/text_converter.h" #include "exec/text_converter.hpp" #include "gen_cpp/PlanNodes_types.h" #include "runtime/runtime_state.h" #include "util/runtime_profile.h" #include "util/types.h" +#include "vec/columns/column.h" #include "vec/common/string_ref.h" #include "vec/core/types.h" namespace doris::vectorized { @@ -258,73 +262,112 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, std::vector columns(_slot_num); bool schema_eos = false; - do { - bool mem_reuse = block->mem_reuse(); - DCHECK(block->rows() == 0); + if (_schema_scanner->type() == TSchemaTableType::SCH_TABLES) { + do { + bool mem_reuse = block->mem_reuse(); + DCHECK(block->rows() == 0); - columns.resize(_slot_num); - for (int i = 0; i < _slot_num; ++i) { - if (mem_reuse) { - columns[i] = std::move(*block->get_by_position(i).column).mutate(); - } else { - columns[i] = _dest_tuple_desc->slots()[i]->get_empty_mutable_column(); + if (!mem_reuse) { + for (int i = 0; i < _slot_num; ++i) { + int j = _index_map[i]; + const auto slot_desc = _src_tuple_desc->slots()[j]; + block->insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), + slot_desc->get_data_type_ptr(), + slot_desc->col_name())); + } } - } - while (true) { - RETURN_IF_CANCELLED(state); - - // get all slots from schema table. - RETURN_IF_ERROR(_schema_scanner->get_next_row(_src_single_tuple, _tuple_pool.get(), - &schema_eos)); - if (schema_eos) { - *eos = true; - break; + + while (true) { + RETURN_IF_CANCELLED(state); + + // get all slots from schema table. + RETURN_IF_ERROR(_schema_scanner->get_next_block(block, &schema_eos)); + + if (schema_eos) { + *eos = true; + break; + } + + if (block->rows() == state->batch_size()) { + break; + } } - // tuple project - project_tuple(); + if (block->rows()) { + RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, + _dest_tuple_desc->slots().size())); + VLOG_ROW << "VSchemaScanNode output rows: " << block->rows(); + } + } while (block->rows() == 0 && !(*eos)); + } else { + do { + bool mem_reuse = block->mem_reuse(); + DCHECK(block->rows() == 0); + + columns.resize(_slot_num); for (int i = 0; i < _slot_num; ++i) { - auto slot_desc = _dest_tuple_desc->slots()[i]; - if (!slot_desc->is_materialized()) { - continue; + if (mem_reuse) { + columns[i] = std::move(*block->get_by_position(i).column).mutate(); + } else { + columns[i] = _dest_tuple_desc->slots()[i]->get_empty_mutable_column(); } + } + while (true) { + RETURN_IF_CANCELLED(state); + + // get all slots from schema table. + RETURN_IF_ERROR(_schema_scanner->get_next_row(_src_single_tuple, _tuple_pool.get(), + &schema_eos)); + if (schema_eos) { + *eos = true; + break; + } + // tuple project + project_tuple(); + + for (int i = 0; i < _slot_num; ++i) { + auto slot_desc = _dest_tuple_desc->slots()[i]; + if (!slot_desc->is_materialized()) { + continue; + } - if (_dest_single_tuple->is_null(slot_desc->null_indicator_offset())) { - if (slot_desc->is_nullable()) { - auto* nullable_column = - reinterpret_cast(columns[i].get()); - nullable_column->insert_data(nullptr, 0); + if (_dest_single_tuple->is_null(slot_desc->null_indicator_offset())) { + if (slot_desc->is_nullable()) { + auto* nullable_column = + reinterpret_cast(columns[i].get()); + nullable_column->insert_data(nullptr, 0); + } else { + return Status::InternalError( + "nonnull column contains NULL. table={}, column={}", + _table_name, slot_desc->col_name()); + } } else { - return Status::InternalError( - "nonnull column contains NULL. table={}, column={}", _table_name, - slot_desc->col_name()); + RETURN_IF_ERROR(write_slot_to_vectorized_column( + _dest_single_tuple->get_slot(slot_desc->tuple_offset()), slot_desc, + &columns[i])); } - } else { - RETURN_IF_ERROR(write_slot_to_vectorized_column( - _dest_single_tuple->get_slot(slot_desc->tuple_offset()), slot_desc, - &columns[i])); + } + if (columns[0]->size() == state->batch_size()) { + break; } } - if (columns[0]->size() == state->batch_size()) { - break; - } - } - if (!columns.empty() && !columns[0]->empty()) { - auto n_columns = 0; - if (!mem_reuse) { - for (const auto slot_desc : _dest_tuple_desc->slots()) { - block->insert(ColumnWithTypeAndName(std::move(columns[n_columns++]), - slot_desc->get_data_type_ptr(), - slot_desc->col_name())); + if (!columns.empty() && !columns[0]->empty()) { + auto n_columns = 0; + if (!mem_reuse) { + for (const auto slot_desc : _dest_tuple_desc->slots()) { + block->insert(ColumnWithTypeAndName(std::move(columns[n_columns++]), + slot_desc->get_data_type_ptr(), + slot_desc->col_name())); + } + } else { + columns.clear(); } - } else { - columns.clear(); + RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, + _dest_tuple_desc->slots().size())); + VLOG_ROW << "VSchemaScanNode output rows: " << block->rows(); } - RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, - _dest_tuple_desc->slots().size())); - VLOG_ROW << "VSchemaScanNode output rows: " << block->rows(); - } - } while (block->rows() == 0 && !(*eos)); + } while (block->rows() == 0 && !(*eos)); + } reached_limit(block, eos); return Status::OK(); From bb01d2b72932e34f57d293ee235877d7c2e08a4c Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Mon, 9 Jan 2023 10:56:20 +0800 Subject: [PATCH 02/24] clang-format --- be/src/exec/schema_scanner/schema_tables_scanner.cpp | 3 +-- be/src/vec/exec/vschema_scan_node.cpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index b52d0ae608d1e9..7744d31db53620 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "exec/schema_scanner/schema_tables_scanner.h" - #include "common/status.h" #include "exec/schema_scanner/schema_helper.h" +#include "exec/schema_scanner/schema_tables_scanner.h" #include "runtime/primitive_type.h" #include "vec/columns/column_complex.h" #include "vec/common/string_ref.h" diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index ee86589c5ba187..168aab47a4aa5f 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -17,8 +17,6 @@ #include "vec/exec/vschema_scan_node.h" -#include - #include "common/status.h" #include "exec/text_converter.h" #include "exec/text_converter.hpp" @@ -29,6 +27,7 @@ #include "vec/columns/column.h" #include "vec/common/string_ref.h" #include "vec/core/types.h" +#include "vec/exec/vschema_scan_node.h" namespace doris::vectorized { VSchemaScanNode::VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, From 5f54c42e2907dcb9623feed47c6e817a389b09da Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Mon, 9 Jan 2023 10:59:33 +0800 Subject: [PATCH 03/24] clang-format 2 --- be/src/exec/schema_scanner/schema_tables_scanner.cpp | 3 ++- be/src/vec/exec/vschema_scan_node.cpp | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index 7744d31db53620..b52d0ae608d1e9 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -15,9 +15,10 @@ // specific language governing permissions and limitations // under the License. +#include "exec/schema_scanner/schema_tables_scanner.h" + #include "common/status.h" #include "exec/schema_scanner/schema_helper.h" -#include "exec/schema_scanner/schema_tables_scanner.h" #include "runtime/primitive_type.h" #include "vec/columns/column_complex.h" #include "vec/common/string_ref.h" diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index 168aab47a4aa5f..a96aae7b64bf5e 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -27,7 +27,6 @@ #include "vec/columns/column.h" #include "vec/common/string_ref.h" #include "vec/core/types.h" -#include "vec/exec/vschema_scan_node.h" namespace doris::vectorized { VSchemaScanNode::VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, From f1d4f4ecee4fce054f1bc6ac17000dedd3fe44da Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Mon, 9 Jan 2023 17:25:26 +0800 Subject: [PATCH 04/24] fix --- be/src/exec/schema_scanner.cpp | 154 +++++ be/src/exec/schema_scanner.h | 1 + .../schema_scanner/schema_tables_scanner.cpp | 537 ++++-------------- .../schema_scanner/schema_tables_scanner.h | 6 +- be/src/vec/exec/vschema_scan_node.cpp | 17 +- 5 files changed, 286 insertions(+), 429 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 436e6b8fd0b7ba..67c36c2fca7976 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -245,4 +245,158 @@ Status SchemaScanner::create_tuple_desc(ObjectPool* pool) { return Status::OK(); } +Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, + const SlotDescriptor* slot_desc) { + if (!block->has(slot_desc->col_name())) { + return Status::OK(); + } + vectorized::IColumn* col_ptr = const_cast( + block->get_by_name(slot_desc->col_name()).column.get()); + + if (data == nullptr) { + auto* nullable_column = reinterpret_cast(col_ptr); + nullable_column->get_null_map_data().push_back(1); + nullable_column->insert_data(nullptr, 0); + return Status::OK(); + } + if (slot_desc->is_nullable()) { + auto* nullable_column = reinterpret_cast(col_ptr); + nullable_column->get_null_map_data().push_back(0); + col_ptr = &nullable_column->get_nested_column(); + } + switch (slot_desc->type().type) { + case TYPE_HLL: { + HyperLogLog* hll_slot = reinterpret_cast(data); + reinterpret_cast(col_ptr)->get_data().emplace_back(*hll_slot); + break; + } + case TYPE_VARCHAR: + case TYPE_CHAR: + case TYPE_STRING: { + StringValue* str_slot = reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data(str_slot->ptr, + str_slot->len); + break; + } + + case TYPE_BOOLEAN: { + uint8_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_TINYINT: { + int8_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_SMALLINT: { + int16_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_INT: { + int32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_BIGINT: { + int64_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_LARGEINT: { + __int128 num; + memcpy(&num, data, sizeof(__int128)); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_FLOAT: { + float num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_DOUBLE: { + double num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_DATE: { + vectorized::VecDateTimeValue value; + DateTimeValue* ts_slot = reinterpret_cast(data); + value.convert_dt_to_vec_dt(ts_slot); + reinterpret_cast*>(col_ptr)->insert_data( + reinterpret_cast(&value), 0); + break; + } + + case TYPE_DATEV2: { + uint32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_DATETIME: { + vectorized::VecDateTimeValue value; + DateTimeValue* ts_slot = reinterpret_cast(data); + value.convert_dt_to_vec_dt(ts_slot); + reinterpret_cast*>(col_ptr)->insert_data( + reinterpret_cast(&value), 0); + break; + } + + case TYPE_DATETIMEV2: { + uint32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_DECIMALV2: { + const vectorized::Int128 num = (reinterpret_cast(data))->value; + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + case TYPE_DECIMAL128I: { + const vectorized::Int128 num = (reinterpret_cast(data))->value; + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + case TYPE_DECIMAL32: { + const int32_t num = *reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + case TYPE_DECIMAL64: { + const int64_t num = *reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + default: { + DCHECK(false) << "bad slot type: " << slot_desc->type(); + std::stringstream ss; + ss << "Fail to convert schema type:'" << slot_desc->type() << " on column:`" + << slot_desc->col_name() + "`"; + return Status::InternalError(ss.str()); + } + } + + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index d9c9076796fc3c..c40c10d0cad165 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -97,6 +97,7 @@ class SchemaScanner { Status create_tuple_desc(ObjectPool* pool); Status create_columns(const std::vector* table_structure, ObjectPool* pool); + Status fill_dest_column(vectorized::Block* block, void* data, const SlotDescriptor* slot_desc); bool _is_init; // this is used for sub class diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index b52d0ae608d1e9..9f9c84f2181ff0 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -53,8 +53,7 @@ SchemaScanner::ColumnDesc SchemaTablesScanner::_s_tbls_columns[] = { SchemaTablesScanner::SchemaTablesScanner() : SchemaScanner(_s_tbls_columns, sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _db_index(0), - _table_index(0) { + _db_index(0) { _schema_table_type = TSchemaTableType::SCH_TABLES; } @@ -92,172 +91,6 @@ Status SchemaTablesScanner::start(RuntimeState* state) { } Status SchemaTablesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - const TTableStatus& tbl_status = _table_result.tables[_table_index]; - // catalog - { - if (!_db_result.__isset.catalogs) { - tuple->set_null(_tuple_desc->slots()[0]->null_indicator_offset()); - } else { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string catalog_name = _db_result.catalogs[_db_index - 1]; - str_slot->data = (char*)pool->allocate(catalog_name.size()); - str_slot->size = catalog_name.size(); - memcpy(const_cast(str_slot->data), catalog_name.c_str(), str_slot->size); - } - } - // schema - { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); - str_slot->data = (char*)pool->allocate(db_name.size()); - str_slot->size = db_name.size(); - memcpy(const_cast(str_slot->data), db_name.c_str(), str_slot->size); - } - // name - { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.name; - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - } - // type - { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.type; - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - } - // engine - if (tbl_status.__isset.engine) { - void* slot = tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.engine; - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - } else { - tuple->set_null(_tuple_desc->slots()[4]->null_indicator_offset()); - } - // version - { tuple->set_null(_tuple_desc->slots()[5]->null_indicator_offset()); } - // row_format - { tuple->set_null(_tuple_desc->slots()[6]->null_indicator_offset()); } - // rows - if (tbl_status.__isset.rows) { - void* slot = tuple->get_slot(_tuple_desc->slots()[7]->tuple_offset()); - *(reinterpret_cast(slot)) = tbl_status.rows; - } else { - tuple->set_null(_tuple_desc->slots()[7]->null_indicator_offset()); - } - // avg_row_length - if (tbl_status.__isset.avg_row_length) { - void* slot = tuple->get_slot(_tuple_desc->slots()[8]->tuple_offset()); - *(reinterpret_cast(slot)) = tbl_status.avg_row_length; - } else { - tuple->set_null(_tuple_desc->slots()[8]->null_indicator_offset()); - } - // data_length - if (tbl_status.__isset.avg_row_length) { - void* slot = tuple->get_slot(_tuple_desc->slots()[9]->tuple_offset()); - *(reinterpret_cast(slot)) = tbl_status.data_length; - } else { - tuple->set_null(_tuple_desc->slots()[9]->null_indicator_offset()); - } // max_data_length - { tuple->set_null(_tuple_desc->slots()[10]->null_indicator_offset()); } - // index_length - { tuple->set_null(_tuple_desc->slots()[11]->null_indicator_offset()); } - // data_free - { tuple->set_null(_tuple_desc->slots()[12]->null_indicator_offset()); } - // auto_increment - { tuple->set_null(_tuple_desc->slots()[13]->null_indicator_offset()); } - // creation_time - if (tbl_status.__isset.create_time) { - int64_t create_time = tbl_status.create_time; - if (create_time <= 0) { - tuple->set_null(_tuple_desc->slots()[14]->null_indicator_offset()); - } else { - tuple->set_not_null(_tuple_desc->slots()[14]->null_indicator_offset()); - void* slot = tuple->get_slot(_tuple_desc->slots()[14]->tuple_offset()); - DateTimeValue* time_slot = reinterpret_cast(slot); - time_slot->from_unixtime(create_time, TimezoneUtils::default_time_zone); - } - } - // update_time - if (tbl_status.__isset.update_time) { - int64_t update_time = tbl_status.update_time; - if (update_time <= 0) { - tuple->set_null(_tuple_desc->slots()[15]->null_indicator_offset()); - } else { - tuple->set_not_null(_tuple_desc->slots()[15]->null_indicator_offset()); - void* slot = tuple->get_slot(_tuple_desc->slots()[15]->tuple_offset()); - DateTimeValue* time_slot = reinterpret_cast(slot); - time_slot->from_unixtime(update_time, TimezoneUtils::default_time_zone); - } - } - // check_time - if (tbl_status.__isset.last_check_time) { - int64_t check_time = tbl_status.last_check_time; - if (check_time <= 0) { - tuple->set_null(_tuple_desc->slots()[16]->null_indicator_offset()); - } else { - tuple->set_not_null(_tuple_desc->slots()[16]->null_indicator_offset()); - void* slot = tuple->get_slot(_tuple_desc->slots()[16]->tuple_offset()); - DateTimeValue* time_slot = reinterpret_cast(slot); - time_slot->from_unixtime(check_time, TimezoneUtils::default_time_zone); - } - } - // collation - if (tbl_status.__isset.collation) { - void* slot = tuple->get_slot(_tuple_desc->slots()[17]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.collation; - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - } else { - tuple->set_null(_tuple_desc->slots()[17]->null_indicator_offset()); - } - // checksum - { tuple->set_null(_tuple_desc->slots()[18]->null_indicator_offset()); } - // create_options - { tuple->set_null(_tuple_desc->slots()[19]->null_indicator_offset()); } - // create_comment - { - void* slot = tuple->get_slot(_tuple_desc->slots()[20]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.comment; - str_slot->size = src->length(); - if (str_slot->size == 0) { - str_slot->data = nullptr; - } else { - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - } - } - _table_index++; return Status::OK(); } @@ -288,7 +121,6 @@ Status SchemaTablesScanner::get_new_table() { } else { return Status::InternalError("IP or port doesn't exists"); } - _table_index = 0; return Status::OK(); } @@ -299,295 +131,172 @@ Status SchemaTablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) if (nullptr == tuple || nullptr == pool || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - while (_table_index >= _table_result.tables.size()) { - if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); - } else { - *eos = true; - return Status::OK(); - } - } - *eos = false; return fill_one_row(tuple, pool); } -Status SchemaTablesScanner::fill_dest_column(vectorized::Block* block, void* data, - const SlotDescriptor* slot_desc) { - if (!block->has(slot_desc->col_name())) { - return Status::OK(); - } - vectorized::IColumn* col_ptr = const_cast( - block->get_by_name(slot_desc->col_name()).column.get()); - - if (data == nullptr) { - auto* nullable_column = reinterpret_cast(col_ptr); - nullable_column->get_null_map_data().push_back(1); - nullable_column->insert_data(nullptr, 0); - return Status::OK(); - } - if (slot_desc->is_nullable()) { - auto* nullable_column = reinterpret_cast(col_ptr); - nullable_column->get_null_map_data().push_back(0); - // if (data == nullptr) { - // nullable_column->insert_data(nullptr, 0); - // return Status::OK(); - // } - col_ptr = &nullable_column->get_nested_column(); - } - switch (slot_desc->type().type) { - case TYPE_HLL: { - HyperLogLog* hll_slot = reinterpret_cast(data); - reinterpret_cast(col_ptr)->get_data().emplace_back(*hll_slot); - break; - } - case TYPE_VARCHAR: - case TYPE_CHAR: - case TYPE_STRING: { - StringRef* str_slot = reinterpret_cast(data); - reinterpret_cast(col_ptr)->insert_data(str_slot->data, - str_slot->size); - break; - } - - case TYPE_BOOLEAN: { - uint8_t num = *reinterpret_cast(data); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_TINYINT: { - int8_t num = *reinterpret_cast(data); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_SMALLINT: { - int16_t num = *reinterpret_cast(data); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_INT: { - int32_t num = *reinterpret_cast(data); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_BIGINT: { - int64_t num = *reinterpret_cast(data); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_LARGEINT: { - __int128 num; - memcpy(&num, data, sizeof(__int128)); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_FLOAT: { - float num = *reinterpret_cast(data); - reinterpret_cast*>(col_ptr)->insert_value( - num); - break; - } - - case TYPE_DOUBLE: { - double num = *reinterpret_cast(data); - reinterpret_cast*>(col_ptr)->insert_value( - num); - break; - } - - case TYPE_DATE: { - vectorized::VecDateTimeValue value; - DateTimeValue* ts_slot = reinterpret_cast(data); - value.convert_dt_to_vec_dt(ts_slot); - reinterpret_cast*>(col_ptr)->insert_data( - reinterpret_cast(&value), 0); - break; - } - - case TYPE_DATEV2: { - uint32_t num = *reinterpret_cast(data); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_DATETIME: { - vectorized::VecDateTimeValue value; - DateTimeValue* ts_slot = reinterpret_cast(data); - value.convert_dt_to_vec_dt(ts_slot); - reinterpret_cast*>(col_ptr)->insert_data( - reinterpret_cast(&value), 0); - break; - } - - case TYPE_DATETIMEV2: { - uint32_t num = *reinterpret_cast(data); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_DECIMALV2: { - const vectorized::Int128 num = (reinterpret_cast(data))->value; - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - case TYPE_DECIMAL128I: { - const vectorized::Int128 num = (reinterpret_cast(data))->value; - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - case TYPE_DECIMAL32: { - const int32_t num = *reinterpret_cast(data); - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - case TYPE_DECIMAL64: { - const int64_t num = *reinterpret_cast(data); - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - default: { - DCHECK(false) << "bad slot type: " << slot_desc->type(); - std::stringstream ss; - ss << "Fail to convert schema type:'" << slot_desc->type() << " on column:`" - << slot_desc->col_name() + "`"; - return Status::InternalError(ss.str()); - } - } - - return Status::OK(); -} - -Status SchemaTablesScanner::fill_one_row(vectorized::Block* block) { - const TTableStatus& tbl_status = _table_result.tables[_table_index]; +Status SchemaTablesScanner::fill_block_imp(vectorized::Block* block) { + auto table_num = _table_result.tables.size(); // catalog { std::string catalog_name = _db_result.catalogs[_db_index - 1]; StringRef str_slot = StringRef(catalog_name.c_str(), catalog_name.size()); - fill_dest_column(block, &str_slot, _tuple_desc->slots()[0]); + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, &str_slot, _tuple_desc->slots()[0]); + } } // schema { std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); StringRef str_slot = StringRef(db_name.c_str(), db_name.size()); - fill_dest_column(block, &str_slot, _tuple_desc->slots()[1]); + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, &str_slot, _tuple_desc->slots()[1]); + } } // name - { - const std::string* src = &tbl_status.name; + for (int i = 0; i < table_num; ++i) { + const std::string* src = &_table_result.tables[i].name; StringRef str_slot = StringRef(src->c_str(), src->size()); fill_dest_column(block, &str_slot, _tuple_desc->slots()[2]); } // type - { - const std::string* src = &tbl_status.type; + for (int i = 0; i < table_num; ++i) { + const std::string* src = &_table_result.tables[i].type; StringRef str_slot = StringRef(src->c_str(), src->size()); fill_dest_column(block, &str_slot, _tuple_desc->slots()[3]); } // engine - if (tbl_status.__isset.engine) { - const std::string* src = &tbl_status.engine; - StringRef str_slot = StringRef(src->c_str(), src->size()); - fill_dest_column(block, &str_slot, _tuple_desc->slots()[4]); - } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[4]); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.engine) { + const std::string* src = &tbl_status.engine; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _tuple_desc->slots()[4]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[4]); + } } // version - { fill_dest_column(block, nullptr, _tuple_desc->slots()[5]); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[5]); + } // row_format - { fill_dest_column(block, nullptr, _tuple_desc->slots()[6]); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[6]); + } // rows - if (tbl_status.__isset.rows) { - int64_t src = tbl_status.rows; - fill_dest_column(block, &src, _tuple_desc->slots()[7]); - } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[7]); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.rows) { + int64_t src = tbl_status.rows; + fill_dest_column(block, &src, _tuple_desc->slots()[7]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[7]); + } } // avg_row_length - if (tbl_status.__isset.avg_row_length) { - int64_t src = tbl_status.avg_row_length; - fill_dest_column(block, &src, _tuple_desc->slots()[8]); - } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[8]); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.avg_row_length) { + int64_t src = tbl_status.avg_row_length; + fill_dest_column(block, &src, _tuple_desc->slots()[8]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[8]); + } } // data_length - if (tbl_status.__isset.avg_row_length) { - int64_t src = tbl_status.data_length; - fill_dest_column(block, &src, _tuple_desc->slots()[9]); - } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[9]); - } // max_data_length - { fill_dest_column(block, nullptr, _tuple_desc->slots()[10]); } + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.avg_row_length) { + int64_t src = tbl_status.data_length; + fill_dest_column(block, &src, _tuple_desc->slots()[9]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[9]); + } + } + // max_data_length + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[10]); + } // index_length - { fill_dest_column(block, nullptr, _tuple_desc->slots()[11]); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[11]); + } // data_free - { fill_dest_column(block, nullptr, _tuple_desc->slots()[12]); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[12]); + } // auto_increment - { fill_dest_column(block, nullptr, _tuple_desc->slots()[13]); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[13]); + } // creation_time - if (tbl_status.__isset.create_time) { - int64_t create_time = tbl_status.create_time; - if (create_time <= 0) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[14]); - } else { - DateTimeValue time_slot; - time_slot.from_unixtime(create_time, TimezoneUtils::default_time_zone); - fill_dest_column(block, &time_slot, _tuple_desc->slots()[14]); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.create_time) { + int64_t create_time = tbl_status.create_time; + if (create_time <= 0) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[14]); + } else { + DateTimeValue time_slot; + time_slot.from_unixtime(create_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, &time_slot, _tuple_desc->slots()[14]); + } } } // update_time - if (tbl_status.__isset.update_time) { - int64_t update_time = tbl_status.update_time; - if (update_time <= 0) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[15]); - } else { - DateTimeValue time_slot; - time_slot.from_unixtime(update_time, TimezoneUtils::default_time_zone); - fill_dest_column(block, &time_slot, _tuple_desc->slots()[15]); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.update_time) { + int64_t update_time = tbl_status.update_time; + if (update_time <= 0) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[15]); + } else { + DateTimeValue time_slot; + time_slot.from_unixtime(update_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, &time_slot, _tuple_desc->slots()[15]); + } } } // check_time - if (tbl_status.__isset.last_check_time) { - int64_t check_time = tbl_status.last_check_time; - if (check_time <= 0) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[16]); - } else { - DateTimeValue time_slot; - time_slot.from_unixtime(check_time, TimezoneUtils::default_time_zone); - fill_dest_column(block, &time_slot, _tuple_desc->slots()[16]); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.last_check_time) { + int64_t check_time = tbl_status.last_check_time; + if (check_time <= 0) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[16]); + } else { + DateTimeValue time_slot; + time_slot.from_unixtime(check_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, &time_slot, _tuple_desc->slots()[16]); + } } } // collation - if (tbl_status.__isset.collation) { - const std::string* src = &tbl_status.collation; - StringRef str_slot = StringRef(src->c_str(), src->size()); - fill_dest_column(block, &str_slot, _tuple_desc->slots()[17]); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.collation) { + const std::string* src = &tbl_status.collation; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _tuple_desc->slots()[17]); - } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[17]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[17]); + } } // checksum - { fill_dest_column(block, nullptr, _tuple_desc->slots()[18]); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[18]); + } // create_options - { fill_dest_column(block, nullptr, _tuple_desc->slots()[19]); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[19]); + } // create_comment - { - const std::string* src = &tbl_status.comment; + for (int i = 0; i < table_num; ++i) { + const std::string* src = &_table_result.tables[i].comment; StringRef str_slot = StringRef(src->c_str(), src->size()); fill_dest_column(block, &str_slot, _tuple_desc->slots()[20]); } - _table_index++; return Status::OK(); } @@ -598,16 +307,14 @@ Status SchemaTablesScanner::get_next_block(vectorized::Block* block, bool* eos) if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - while (_table_index >= _table_result.tables.size()) { - if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); - } else { - *eos = true; - return Status::OK(); - } + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(get_new_table()); + } else { + *eos = true; + return Status::OK(); } *eos = false; - return fill_one_row(block); + return fill_block_imp(block); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h index 3375f72d7b4036..8918b17e0093be 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_tables_scanner.h @@ -31,16 +31,14 @@ class SchemaTablesScanner : public SchemaScanner { virtual Status start(RuntimeState* state); virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); - virtual Status get_next_block(vectorized::Block* block, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: Status get_new_table(); Status fill_one_row(Tuple* tuple, MemPool* pool); - Status fill_one_row(vectorized::Block* block); - Status fill_dest_column(vectorized::Block* block, void* data, const SlotDescriptor* slot_desc); + Status fill_block_imp(vectorized::Block* block); int _db_index; - int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; static SchemaScanner::ColumnDesc _s_tbls_columns[]; diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index a96aae7b64bf5e..080724cc396a86 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -262,17 +262,14 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, if (_schema_scanner->type() == TSchemaTableType::SCH_TABLES) { do { - bool mem_reuse = block->mem_reuse(); - DCHECK(block->rows() == 0); + block->clear(); - if (!mem_reuse) { - for (int i = 0; i < _slot_num; ++i) { - int j = _index_map[i]; - const auto slot_desc = _src_tuple_desc->slots()[j]; - block->insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), - slot_desc->get_data_type_ptr(), - slot_desc->col_name())); - } + for (int i = 0; i < _slot_num; ++i) { + int j = _index_map[i]; + const auto slot_desc = _src_tuple_desc->slots()[j]; + block->insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), + slot_desc->get_data_type_ptr(), + slot_desc->col_name())); } while (true) { From 471102bd9eafc0a0534666ffb78a660bab2e0ddf Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Tue, 10 Jan 2023 15:44:03 +0800 Subject: [PATCH 05/24] change schema_backends_scanner --- .../schema_backends_scanner.cpp | 53 +++++++++++++++++++ .../schema_scanner/schema_backends_scanner.h | 3 ++ .../schema_scanner/schema_tables_scanner.cpp | 11 ++-- .../schema_scanner/schema_tables_scanner.h | 2 +- be/src/vec/exec/vschema_scan_node.cpp | 3 +- 5 files changed, 64 insertions(+), 8 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.cpp b/be/src/exec/schema_scanner/schema_backends_scanner.cpp index 9c1b0c015a25f9..0b99c4afeb6a93 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_backends_scanner.cpp @@ -21,6 +21,7 @@ #include #include +#include "common/status.h" #include "exec/schema_scanner.h" #include "gen_cpp/FrontendService.h" #include "runtime/client_cache.h" @@ -59,6 +60,58 @@ Status SchemaBackendsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eo return _fill_one_row(tuple, pool); } +Status SchemaBackendsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + *eos = true; + return _fill_block_imp(block); +} + +Status SchemaBackendsScanner::_fill_block_imp(vectorized::Block* block) { + auto row_num = _batch_data.size(); + for (size_t col_idx = 0; col_idx < _column_num; ++col_idx) { + auto it = _col_name_to_type.find(_columns[col_idx].name); + if (it == _col_name_to_type.end()) { + if (_columns[col_idx].is_null) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[col_idx]); + } + } else { + return Status::InternalError( + "column {} is not found in BE, and {} is not nullable.", + _columns[col_idx].name, _columns[col_idx].name); + } + } else if (it->second == TYPE_BIGINT) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].longVal, + _tuple_desc->slots()[col_idx]); + } + } else if (it->second == TYPE_INT) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].intVal, + _tuple_desc->slots()[col_idx]); + } + } else if (it->second == TYPE_VARCHAR) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].stringVal, + _tuple_desc->slots()[col_idx]); + } + } else if (it->second == TYPE_DOUBLE) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].doubleVal, + _tuple_desc->slots()[col_idx]); + } + } else { + // other type + } + } + return Status::OK(); +} + Status SchemaBackendsScanner::_fill_one_row(Tuple* tuple, MemPool* pool) { memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); for (size_t col_idx = 0; col_idx < _column_num; ++col_idx) { diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.h b/be/src/exec/schema_scanner/schema_backends_scanner.h index 32753f568e5b18..8314d6d5cec231 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.h +++ b/be/src/exec/schema_scanner/schema_backends_scanner.h @@ -17,6 +17,7 @@ #pragma once +#include "common/status.h" #include "exec/schema_scanner.h" namespace doris { @@ -27,9 +28,11 @@ class SchemaBackendsScanner : public SchemaScanner { Status start(RuntimeState* state) override; Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: Status _fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_imp(vectorized::Block* block); Status _fetch_backends_info(); Status _fill_one_col(Tuple* tuple, MemPool* pool, size_t idx); Status _set_col_name_to_type(); diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index 9f9c84f2181ff0..9f422a1c76422b 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -52,10 +52,9 @@ SchemaScanner::ColumnDesc SchemaTablesScanner::_s_tbls_columns[] = { SchemaTablesScanner::SchemaTablesScanner() : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _db_index(0) { - _schema_table_type = TSchemaTableType::SCH_TABLES; -} + sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_TABLES), + _db_index(0) {} SchemaTablesScanner::~SchemaTablesScanner() {} @@ -134,7 +133,7 @@ Status SchemaTablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) return fill_one_row(tuple, pool); } -Status SchemaTablesScanner::fill_block_imp(vectorized::Block* block) { +Status SchemaTablesScanner::_fill_block_imp(vectorized::Block* block) { auto table_num = _table_result.tables.size(); // catalog { @@ -314,7 +313,7 @@ Status SchemaTablesScanner::get_next_block(vectorized::Block* block, bool* eos) return Status::OK(); } *eos = false; - return fill_block_imp(block); + return _fill_block_imp(block); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h index 8918b17e0093be..638eaac11e62de 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_tables_scanner.h @@ -36,7 +36,7 @@ class SchemaTablesScanner : public SchemaScanner { private: Status get_new_table(); Status fill_one_row(Tuple* tuple, MemPool* pool); - Status fill_block_imp(vectorized::Block* block); + Status _fill_block_imp(vectorized::Block* block); int _db_index; TGetDbsResult _db_result; diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index 080724cc396a86..fbbcb2d0119ccf 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -260,7 +260,8 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, std::vector columns(_slot_num); bool schema_eos = false; - if (_schema_scanner->type() == TSchemaTableType::SCH_TABLES) { + if (_schema_scanner->type() == TSchemaTableType::SCH_TABLES || + _schema_scanner->type() == TSchemaTableType::SCH_BACKENDS) { do { block->clear(); From 8dffccee4fe761abe2834d3339ca63aa80b676de Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Tue, 10 Jan 2023 16:32:03 +0800 Subject: [PATCH 06/24] modify charsets, collations --- .../schema_charsets_scanner.cpp | 47 +++++++++++++++- .../schema_scanner/schema_charsets_scanner.h | 2 + .../schema_collations_scanner.cpp | 53 +++++++++++++++++++ .../schema_collations_scanner.h | 2 + 4 files changed, 102 insertions(+), 2 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp index 0a96db94c4347d..7eaa90ebffba8e 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp @@ -17,8 +17,8 @@ #include "exec/schema_scanner/schema_charsets_scanner.h" +#include "common/status.h" #include "vec/common/string_ref.h" - namespace doris { SchemaScanner::ColumnDesc SchemaCharsetsScanner::_s_css_columns[] = { @@ -35,7 +35,8 @@ SchemaCharsetsScanner::CharsetStruct SchemaCharsetsScanner::_s_charsets[] = { }; SchemaCharsetsScanner::SchemaCharsetsScanner() - : SchemaScanner(_s_css_columns, sizeof(_s_css_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_css_columns, sizeof(_s_css_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_CHARSETS), _index(0) {} SchemaCharsetsScanner::~SchemaCharsetsScanner() {} @@ -101,4 +102,46 @@ Status SchemaCharsetsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eo return fill_one_row(tuple, pool); } +Status SchemaCharsetsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("call this before initial."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("invalid parameter."); + } + *eos = true; + return _fill_block_imp(block); +} + +Status SchemaCharsetsScanner::_fill_block_imp(vectorized::Block* block) { + auto row_num = 0; + while (nullptr != _s_charsets[row_num].charset) { + ++row_num; + } + + // variables names + for (int i = 0; i < row_num; ++i) { + StringRef str = StringRef(_s_charsets[i].charset, strlen(_s_charsets[i].charset)); + fill_dest_column(block, &str, _tuple_desc->slots()[0]); + } + // DEFAULT_COLLATE_NAME + for (int i = 0; i < row_num; ++i) { + StringRef str = StringRef(_s_charsets[i].default_collation, + strlen(_s_charsets[i].default_collation)); + fill_dest_column(block, &str, _tuple_desc->slots()[1]); + } + // DESCRIPTION + for (int i = 0; i < row_num; ++i) { + StringRef str = + StringRef(_s_charsets[i].description, strlen(_s_charsets[i].description)); + fill_dest_column(block, &str, _tuple_desc->slots()[2]); + } + // maxlen + for (int i = 0; i < row_num; ++i) { + int64_t src = _s_charsets[i].maxlen; + fill_dest_column(block, &src, _tuple_desc->slots()[3]); + } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.h b/be/src/exec/schema_scanner/schema_charsets_scanner.h index 3a9f07100ae342..91d32f1b8daa1e 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.h @@ -29,6 +29,7 @@ class SchemaCharsetsScanner : public SchemaScanner { virtual ~SchemaCharsetsScanner(); virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: struct CharsetStruct { @@ -39,6 +40,7 @@ class SchemaCharsetsScanner : public SchemaScanner { }; Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_imp(vectorized::Block* block); int _index; static SchemaScanner::ColumnDesc _s_css_columns[]; diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.cpp b/be/src/exec/schema_scanner/schema_collations_scanner.cpp index 3201deb68bfcc6..a1b02ffb9dd55b 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_collations_scanner.cpp @@ -17,6 +17,7 @@ #include "exec/schema_scanner/schema_collations_scanner.h" +#include "common/status.h" #include "runtime/primitive_type.h" #include "vec/common/string_ref.h" @@ -123,4 +124,56 @@ Status SchemaCollationsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* return fill_one_row(tuple, pool); } +Status SchemaCollationsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("call this before initial."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("invalid parameter."); + } + + *eos = true; + return _fill_block_imp(block); +} + +Status SchemaCollationsScanner::_fill_block_imp(vectorized::Block* block) { + auto row_num = 0; + while (nullptr != _s_collations[row_num].name) { + ++row_num; + } + // COLLATION_NAME + for (int i = 0; i < row_num; ++i) { + StringValue str = StringValue(_s_collations[i].name, strlen(_s_collations[i].name)); + fill_dest_column(block, &str, _tuple_desc->slots()[0]); + } + // charset + for (int i = 0; i < row_num; ++i) { + StringValue str = StringValue(_s_collations[i].charset, strlen(_s_collations[i].charset)); + fill_dest_column(block, &str, _tuple_desc->slots()[1]); + } + // id + for (int i = 0; i < row_num; ++i) { + int64_t src = _s_collations[_index].id; + fill_dest_column(block, &src, _tuple_desc->slots()[2]); + } + // is_default + for (int i = 0; i < row_num; ++i) { + StringValue str = + StringValue(_s_collations[i].is_default, strlen(_s_collations[i].is_default)); + fill_dest_column(block, &str, _tuple_desc->slots()[3]); + } + // IS_COMPILED + for (int i = 0; i < row_num; ++i) { + StringValue str = + StringValue(_s_collations[i].is_compile, strlen(_s_collations[i].is_compile)); + fill_dest_column(block, &str, _tuple_desc->slots()[4]); + } + // sortlen + for (int i = 0; i < row_num; ++i) { + int64_t src = _s_collations[_index].sortlen; + fill_dest_column(block, &src, _tuple_desc->slots()[5]); + } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.h b/be/src/exec/schema_scanner/schema_collations_scanner.h index 66e3faeed33e69..a5c00e0d13a893 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.h +++ b/be/src/exec/schema_scanner/schema_collations_scanner.h @@ -29,6 +29,7 @@ class SchemaCollationsScanner : public SchemaScanner { virtual ~SchemaCollationsScanner(); virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: struct CollationStruct { @@ -41,6 +42,7 @@ class SchemaCollationsScanner : public SchemaScanner { }; Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_imp(vectorized::Block* block); int _index; static SchemaScanner::ColumnDesc _s_cols_columns[]; From 565268af4e1b535de99b57f92a07cfac25012c84 Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Tue, 10 Jan 2023 22:13:21 +0800 Subject: [PATCH 07/24] fix bug and modify columns --- .../schema_backends_scanner.cpp | 4 +- .../schema_scanner/schema_backends_scanner.h | 2 +- .../schema_charsets_scanner.cpp | 4 +- .../schema_scanner/schema_charsets_scanner.h | 2 +- .../schema_collations_scanner.cpp | 7 +- .../schema_collations_scanner.h | 2 +- .../schema_scanner/schema_columns_scanner.cpp | 261 +++++++++++++++++- .../schema_scanner/schema_columns_scanner.h | 2 + .../schema_scanner/schema_tables_scanner.cpp | 10 +- .../schema_scanner/schema_tables_scanner.h | 2 +- be/src/vec/exec/vschema_scan_node.cpp | 5 +- 11 files changed, 285 insertions(+), 16 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.cpp b/be/src/exec/schema_scanner/schema_backends_scanner.cpp index 0b99c4afeb6a93..4e903d322a3d76 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_backends_scanner.cpp @@ -68,10 +68,10 @@ Status SchemaBackendsScanner::get_next_block(vectorized::Block* block, bool* eos return Status::InternalError("input pointer is nullptr."); } *eos = true; - return _fill_block_imp(block); + return _fill_block_impl(block); } -Status SchemaBackendsScanner::_fill_block_imp(vectorized::Block* block) { +Status SchemaBackendsScanner::_fill_block_impl(vectorized::Block* block) { auto row_num = _batch_data.size(); for (size_t col_idx = 0; col_idx < _column_num; ++col_idx) { auto it = _col_name_to_type.find(_columns[col_idx].name); diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.h b/be/src/exec/schema_scanner/schema_backends_scanner.h index 8314d6d5cec231..9c5ee4677b598a 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.h +++ b/be/src/exec/schema_scanner/schema_backends_scanner.h @@ -32,7 +32,7 @@ class SchemaBackendsScanner : public SchemaScanner { private: Status _fill_one_row(Tuple* tuple, MemPool* pool); - Status _fill_block_imp(vectorized::Block* block); + Status _fill_block_impl(vectorized::Block* block); Status _fetch_backends_info(); Status _fill_one_col(Tuple* tuple, MemPool* pool, size_t idx); Status _set_col_name_to_type(); diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp index 7eaa90ebffba8e..a48789a28a491e 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp @@ -110,10 +110,10 @@ Status SchemaCharsetsScanner::get_next_block(vectorized::Block* block, bool* eos return Status::InternalError("invalid parameter."); } *eos = true; - return _fill_block_imp(block); + return _fill_block_impl(block); } -Status SchemaCharsetsScanner::_fill_block_imp(vectorized::Block* block) { +Status SchemaCharsetsScanner::_fill_block_impl(vectorized::Block* block) { auto row_num = 0; while (nullptr != _s_charsets[row_num].charset) { ++row_num; diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.h b/be/src/exec/schema_scanner/schema_charsets_scanner.h index 91d32f1b8daa1e..2b2043926578f4 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.h @@ -40,7 +40,7 @@ class SchemaCharsetsScanner : public SchemaScanner { }; Status fill_one_row(Tuple* tuple, MemPool* pool); - Status _fill_block_imp(vectorized::Block* block); + Status _fill_block_impl(vectorized::Block* block); int _index; static SchemaScanner::ColumnDesc _s_css_columns[]; diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.cpp b/be/src/exec/schema_scanner/schema_collations_scanner.cpp index a1b02ffb9dd55b..0d2f5479becf78 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_collations_scanner.cpp @@ -40,7 +40,8 @@ SchemaCollationsScanner::CollationStruct SchemaCollationsScanner::_s_collations[ SchemaCollationsScanner::SchemaCollationsScanner() : SchemaScanner(_s_cols_columns, - sizeof(_s_cols_columns) / sizeof(SchemaScanner::ColumnDesc)), + sizeof(_s_cols_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_COLLATIONS), _index(0) {} SchemaCollationsScanner::~SchemaCollationsScanner() {} @@ -133,10 +134,10 @@ Status SchemaCollationsScanner::get_next_block(vectorized::Block* block, bool* e } *eos = true; - return _fill_block_imp(block); + return _fill_block_impl(block); } -Status SchemaCollationsScanner::_fill_block_imp(vectorized::Block* block) { +Status SchemaCollationsScanner::_fill_block_impl(vectorized::Block* block) { auto row_num = 0; while (nullptr != _s_collations[row_num].name) { ++row_num; diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.h b/be/src/exec/schema_scanner/schema_collations_scanner.h index a5c00e0d13a893..46883001e119d6 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.h +++ b/be/src/exec/schema_scanner/schema_collations_scanner.h @@ -42,7 +42,7 @@ class SchemaCollationsScanner : public SchemaScanner { }; Status fill_one_row(Tuple* tuple, MemPool* pool); - Status _fill_block_imp(vectorized::Block* block); + Status _fill_block_impl(vectorized::Block* block); int _index; static SchemaScanner::ColumnDesc _s_cols_columns[]; diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp index 60a133b6d0c00f..c614e87cff7d20 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp @@ -54,7 +54,8 @@ SchemaScanner::ColumnDesc SchemaColumnsScanner::_s_col_columns[] = { }; SchemaColumnsScanner::SchemaColumnsScanner() - : SchemaScanner(_s_col_columns, sizeof(_s_col_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_col_columns, sizeof(_s_col_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_COLUMNS), _db_index(0), _table_index(0), _column_index(0) {} @@ -534,4 +535,262 @@ Status SchemaColumnsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos return fill_one_row(tuple, pool); } +Status SchemaColumnsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("use this class before inited."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input parameter is nullptr."); + } + + while (_table_index >= _table_result.tables.size()) { + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(get_new_table()); + } else { + *eos = true; + return Status::OK(); + } + } + RETURN_IF_ERROR(get_new_desc()); + + *eos = false; + return _fill_block_impl(block); +} + +Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { + auto columns_num = _desc_result.columns.size(); + + // TABLE_CATALOG + { + if (!_db_result.__isset.catalogs) { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[0]); + } + } else { + std::string catalog_name = _db_result.catalogs[_db_index - 1]; + StringValue str = StringValue(catalog_name.c_str(), catalog_name.size()); + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[0]); + } + } + } + // TABLE_SCHEMA + { + std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); + StringValue str = StringValue(db_name.c_str(), db_name.size()); + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[1]); + } + } + // TABLE_NAME + { + StringValue str = StringValue(_table_result.tables[_table_index - 1].c_str(), + _table_result.tables[_table_index - 1].length()); + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[2]); + } + } + // COLUMN_NAME + { + for (int i = 0; i < columns_num; ++i) { + StringValue str = StringValue(_desc_result.columns[i].columnDesc.columnName.c_str(), + _desc_result.columns[i].columnDesc.columnName.length()); + fill_dest_column(block, &str, _tuple_desc->slots()[3]); + } + } + // ORDINAL_POSITION + { + for (int i = 0; i < columns_num; ++i) { + int64_t src = i + 1; + fill_dest_column(block, &src, _tuple_desc->slots()[4]); + } + } + // COLUMN_DEFAULT + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[5]); + } + } + // IS_NULLABLE + { + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.isAllowNull) { + if (_desc_result.columns[i].columnDesc.isAllowNull) { + StringValue str = StringValue("YES", 3); + fill_dest_column(block, &str, _tuple_desc->slots()[6]); + } else { + StringValue str = StringValue("NO", 2); + fill_dest_column(block, &str, _tuple_desc->slots()[6]); + } + } else { + StringValue str = StringValue("NO", 2); + fill_dest_column(block, &str, _tuple_desc->slots()[6]); + } + } + } + // DATA_TYPE + { + for (int i = 0; i < columns_num; ++i) { + std::string buffer = to_mysql_data_type_string(_desc_result.columns[i].columnDesc); + StringValue str = StringValue(buffer.c_str(), buffer.length()); + fill_dest_column(block, &str, _tuple_desc->slots()[7]); + } + } + // CHARACTER_MAXIMUM_LENGTH + // For string columns, the maximum length in characters. + { + for (int i = 0; i < columns_num; ++i) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (data_type == TPrimitiveType::VARCHAR || data_type == TPrimitiveType::CHAR || + data_type == TPrimitiveType::STRING) { + if (_desc_result.columns[i].columnDesc.__isset.columnLength) { + int64_t src = _desc_result.columns[i].columnDesc.columnLength; + fill_dest_column(block, &src, _tuple_desc->slots()[8]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[8]); + } + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[8]); + } + } + } + // CHARACTER_OCTET_LENGTH + // For string columns, the maximum length in bytes. + { + for (int i = 0; i < columns_num; ++i) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (data_type == TPrimitiveType::VARCHAR || data_type == TPrimitiveType::CHAR || + data_type == TPrimitiveType::STRING) { + if (_desc_result.columns[i].columnDesc.__isset.columnLength) { + int64_t src = _desc_result.columns[i].columnDesc.columnLength * 4; + fill_dest_column(block, &src, _tuple_desc->slots()[9]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[9]); + } + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[9]); + } + } + } + // NUMERIC_PRECISION + { + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.columnPrecision) { + int64_t src = _desc_result.columns[i].columnDesc.columnPrecision; + fill_dest_column(block, &src, _tuple_desc->slots()[10]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[10]); + } + } + } + // NUMERIC_SCALE + { + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.columnScale) { + int64_t src = _desc_result.columns[i].columnDesc.columnScale; + fill_dest_column(block, &src, _tuple_desc->slots()[11]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[11]); + } + } + } + // DATETIME_PRECISION + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[12]); + } + } + // CHARACTER_SET_NAME + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[13]); + } + } + // COLLATION_NAME + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[14]); + } + } + // COLUMN_TYPE + { + for (int i = 0; i < columns_num; ++i) { + std::string buffer = type_to_string(_desc_result.columns[i].columnDesc); + StringValue str = StringValue(buffer.c_str(), buffer.length()); + fill_dest_column(block, &str, _tuple_desc->slots()[15]); + } + } + // COLUMN_KEY + { + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.columnKey) { + StringValue str = + StringValue(_desc_result.columns[i].columnDesc.columnKey.c_str(), + _desc_result.columns[i].columnDesc.columnKey.length()); + fill_dest_column(block, &str, _tuple_desc->slots()[16]); + } else { + StringValue str = StringValue("", 0); + fill_dest_column(block, &str, _tuple_desc->slots()[16]); + } + } + } + // EXTRA + { + for (int i = 0; i < columns_num; ++i) { + StringValue str = StringValue("", 0); + fill_dest_column(block, &str, _tuple_desc->slots()[17]); + } + } + // PRIVILEGES + { + for (int i = 0; i < columns_num; ++i) { + StringValue str = StringValue("", 0); + fill_dest_column(block, &str, _tuple_desc->slots()[18]); + } + } + // COLUMN_COMMENT + { + for (int i = 0; i < columns_num; ++i) { + StringValue str = StringValue(_desc_result.columns[i].comment.c_str(), + _desc_result.columns[i].comment.length()); + fill_dest_column(block, &str, _tuple_desc->slots()[19]); + } + } + // COLUMN_SIZE + { + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.columnLength) { + int64_t src = _desc_result.columns[i].columnDesc.columnLength; + fill_dest_column(block, &src, _tuple_desc->slots()[20]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[20]); + } + } + } + // DECIMAL_DIGITS + { + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.columnScale) { + int64_t src = _desc_result.columns[i].columnDesc.columnScale; + fill_dest_column(block, &src, _tuple_desc->slots()[21]); + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[21]); + } + } + } + // GENERATION_EXPRESSION + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[22]); + } + } + // SRS_ID + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[23]); + } + } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.h b/be/src/exec/schema_scanner/schema_columns_scanner.h index 25afa984d6ed34..adc89ff7452b81 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.h +++ b/be/src/exec/schema_scanner/schema_columns_scanner.h @@ -30,10 +30,12 @@ class SchemaColumnsScanner : public SchemaScanner { virtual ~SchemaColumnsScanner(); virtual Status start(RuntimeState* state); virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: Status get_new_table(); Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); Status get_new_desc(); Status get_create_table(std::string* result); std::string to_mysql_data_type_string(TColumnDesc& desc); diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index 9f422a1c76422b..a7115d847a256e 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -133,15 +133,19 @@ Status SchemaTablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) return fill_one_row(tuple, pool); } -Status SchemaTablesScanner::_fill_block_imp(vectorized::Block* block) { +Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { auto table_num = _table_result.tables.size(); // catalog - { + if (_db_result.__isset.catalogs) { std::string catalog_name = _db_result.catalogs[_db_index - 1]; StringRef str_slot = StringRef(catalog_name.c_str(), catalog_name.size()); for (int i = 0; i < table_num; ++i) { fill_dest_column(block, &str_slot, _tuple_desc->slots()[0]); } + } else { + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[0]); + } } // schema { @@ -313,7 +317,7 @@ Status SchemaTablesScanner::get_next_block(vectorized::Block* block, bool* eos) return Status::OK(); } *eos = false; - return _fill_block_imp(block); + return _fill_block_impl(block); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h index 638eaac11e62de..58d69e75ffcefa 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_tables_scanner.h @@ -36,7 +36,7 @@ class SchemaTablesScanner : public SchemaScanner { private: Status get_new_table(); Status fill_one_row(Tuple* tuple, MemPool* pool); - Status _fill_block_imp(vectorized::Block* block); + Status _fill_block_impl(vectorized::Block* block); int _db_index; TGetDbsResult _db_result; diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index fbbcb2d0119ccf..f9998f3fce3a21 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -261,7 +261,10 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, bool schema_eos = false; if (_schema_scanner->type() == TSchemaTableType::SCH_TABLES || - _schema_scanner->type() == TSchemaTableType::SCH_BACKENDS) { + _schema_scanner->type() == TSchemaTableType::SCH_BACKENDS || + _schema_scanner->type() == TSchemaTableType::SCH_CHARSETS || + _schema_scanner->type() == TSchemaTableType::SCH_COLLATIONS || + _schema_scanner->type() == TSchemaTableType::SCH_COLUMNS) { do { block->clear(); From a9a356a83ee31015a3b4780c28e4f0df714be218 Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Wed, 11 Jan 2023 13:25:44 +0800 Subject: [PATCH 08/24] modify left schema scanner --- be/src/exec/schema_scanner.cpp | 18 +- be/src/exec/schema_scanner.h | 1 - .../schema_backends_scanner.cpp | 61 +--- .../schema_scanner/schema_backends_scanner.h | 5 - .../schema_charsets_scanner.cpp | 70 +--- .../schema_scanner/schema_charsets_scanner.h | 3 - .../schema_collations_scanner.cpp | 98 +----- .../schema_collations_scanner.h | 3 - .../schema_scanner/schema_columns_scanner.cpp | 315 ++---------------- .../schema_scanner/schema_columns_scanner.h | 13 +- .../schema_scanner/schema_dummy_scanner.cpp | 2 +- .../schema_scanner/schema_dummy_scanner.h | 2 +- .../schema_scanner/schema_files_scanner.cpp | 17 +- .../schema_scanner/schema_files_scanner.h | 2 +- .../schema_partitions_scanner.cpp | 17 +- .../schema_partitions_scanner.h | 2 +- .../schema_scanner/schema_rowsets_scanner.cpp | 142 +++++--- .../schema_scanner/schema_rowsets_scanner.h | 10 +- .../schema_schema_privileges_scanner.cpp | 135 ++++---- .../schema_schema_privileges_scanner.h | 8 +- .../schema_schemata_scanner.cpp | 96 +++--- .../schema_scanner/schema_schemata_scanner.h | 5 +- .../schema_table_privileges_scanner.cpp | 154 ++++----- .../schema_table_privileges_scanner.h | 8 +- .../schema_scanner/schema_tables_scanner.cpp | 18 +- .../schema_scanner/schema_tables_scanner.h | 4 +- .../schema_user_privileges_scanner.cpp | 119 +++---- .../schema_user_privileges_scanner.h | 8 +- .../schema_variables_scanner.cpp | 59 ++-- .../schema_scanner/schema_variables_scanner.h | 5 +- .../schema_scanner/schema_views_scanner.cpp | 199 +++++------ .../schema_scanner/schema_views_scanner.h | 7 +- be/src/vec/exec/vschema_scan_node.cpp | 128 ++----- 33 files changed, 546 insertions(+), 1188 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 67c36c2fca7976..f8ce4b1978ffe8 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -71,19 +71,6 @@ Status SchemaScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("used before initialized."); - } - - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input pointer is nullptr."); - } - - *eos = true; - return Status::OK(); -} - Status SchemaScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("used before initialized."); @@ -254,8 +241,11 @@ Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, block->get_by_name(slot_desc->col_name()).column.get()); if (data == nullptr) { + if (!slot_desc->is_nullable()) { + return Status::InternalError("nonnull column contains NULL. column={}", + slot_desc->col_name()); + } auto* nullable_column = reinterpret_cast(col_ptr); - nullable_column->get_null_map_data().push_back(1); nullable_column->insert_data(nullptr, 0); return Status::OK(); } diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index c40c10d0cad165..76059c52bd26e6 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -83,7 +83,6 @@ class SchemaScanner { virtual Status init(SchemaScannerParam* param, ObjectPool* pool); // Start to work virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); virtual Status get_next_block(vectorized::Block* block, bool* eos); // factory function static SchemaScanner* create(TSchemaTableType::type type); diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.cpp b/be/src/exec/schema_scanner/schema_backends_scanner.cpp index 4e903d322a3d76..85b9527ce81bce 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_backends_scanner.cpp @@ -34,7 +34,7 @@ namespace doris { SchemaBackendsScanner::SchemaBackendsScanner() - : SchemaScanner(nullptr, 0, TSchemaTableType::SCH_BACKENDS), _row_idx(0) {} + : SchemaScanner(nullptr, 0, TSchemaTableType::SCH_BACKENDS) {} Status SchemaBackendsScanner::start(RuntimeState* state) { if (!_is_init) { @@ -45,21 +45,6 @@ Status SchemaBackendsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaBackendsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("Used before initialized."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input pointer is nullptr."); - } - if (_row_idx >= _batch_data.size()) { - *eos = true; - return Status::OK(); - } - *eos = false; - return _fill_one_row(tuple, pool); -} - Status SchemaBackendsScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); @@ -112,50 +97,6 @@ Status SchemaBackendsScanner::_fill_block_impl(vectorized::Block* block) { return Status::OK(); } -Status SchemaBackendsScanner::_fill_one_row(Tuple* tuple, MemPool* pool) { - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - for (size_t col_idx = 0; col_idx < _column_num; ++col_idx) { - RETURN_IF_ERROR(_fill_one_col(tuple, pool, col_idx)); - } - ++_row_idx; - return Status::OK(); -} - -Status SchemaBackendsScanner::_fill_one_col(Tuple* tuple, MemPool* pool, size_t col_idx) { - auto it = _col_name_to_type.find(_columns[col_idx].name); - - // if this column is not exist in BE, we fill it with `NULL`. - if (it == _col_name_to_type.end()) { - if (_columns[col_idx].is_null) { - tuple->set_null(_tuple_desc->slots()[col_idx]->null_indicator_offset()); - } else { - return Status::InternalError("column {} is not found in BE, and {} is not nullable.", - _columns[col_idx].name, _columns[col_idx].name); - } - } else if (it->second == TYPE_BIGINT) { - void* slot = tuple->get_slot(_tuple_desc->slots()[col_idx]->tuple_offset()); - *(reinterpret_cast(slot)) = _batch_data[_row_idx].column_value[col_idx].longVal; - } else if (it->second == TYPE_INT) { - void* slot = tuple->get_slot(_tuple_desc->slots()[col_idx]->tuple_offset()); - *(reinterpret_cast(slot)) = _batch_data[_row_idx].column_value[col_idx].intVal; - } else if (it->second == TYPE_VARCHAR) { - void* slot = tuple->get_slot(_tuple_desc->slots()[col_idx]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->data = - (char*)pool->allocate(_batch_data[_row_idx].column_value[col_idx].stringVal.size()); - str_slot->size = _batch_data[_row_idx].column_value[col_idx].stringVal.size(); - memcpy(const_cast(str_slot->data), - _batch_data[_row_idx].column_value[col_idx].stringVal.c_str(), str_slot->size); - } else if (it->second == TYPE_DOUBLE) { - void* slot = tuple->get_slot(_tuple_desc->slots()[col_idx]->tuple_offset()); - *(reinterpret_cast(slot)) = - _batch_data[_row_idx].column_value[col_idx].doubleVal; - } else { - // other type - } - return Status::OK(); -} - Status SchemaBackendsScanner::_fetch_backends_info() { TFetchSchemaTableDataRequest request; request.cluster_name = ""; diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.h b/be/src/exec/schema_scanner/schema_backends_scanner.h index 9c5ee4677b598a..70b2483c16134b 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.h +++ b/be/src/exec/schema_scanner/schema_backends_scanner.h @@ -27,21 +27,16 @@ class SchemaBackendsScanner : public SchemaScanner { ~SchemaBackendsScanner() override = default; Status start(RuntimeState* state) override; - Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status _fill_one_row(Tuple* tuple, MemPool* pool); Status _fill_block_impl(vectorized::Block* block); Status _fetch_backends_info(); - Status _fill_one_col(Tuple* tuple, MemPool* pool, size_t idx); Status _set_col_name_to_type(); -private: // column_name -> type, set by _set_col_name_to_type() std::unordered_map _col_name_to_type; std::vector _batch_data; - size_t _row_idx; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp index a48789a28a491e..9ee6ac8f4da2ee 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp @@ -19,6 +19,7 @@ #include "common/status.h" #include "vec/common/string_ref.h" + namespace doris { SchemaScanner::ColumnDesc SchemaCharsetsScanner::_s_css_columns[] = { @@ -36,72 +37,10 @@ SchemaCharsetsScanner::CharsetStruct SchemaCharsetsScanner::_s_charsets[] = { SchemaCharsetsScanner::SchemaCharsetsScanner() : SchemaScanner(_s_css_columns, sizeof(_s_css_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_CHARSETS), - _index(0) {} + TSchemaTableType::SCH_CHARSETS) {} SchemaCharsetsScanner::~SchemaCharsetsScanner() {} -Status SchemaCharsetsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // variables names - { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_charsets[_index].charset); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_charsets[_index].charset, len + 1); - str_slot->size = len; - } - // DEFAULT_COLLATE_NAME - { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_charsets[_index].default_collation); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_charsets[_index].default_collation, len + 1); - str_slot->size = len; - } - // DESCRIPTION - { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_charsets[_index].description); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_charsets[_index].description, len + 1); - str_slot->size = len; - } - // maxlen - { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - *(int64_t*)slot = _s_charsets[_index].maxlen; - } - _index++; - return Status::OK(); -} - -Status SchemaCharsetsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("call this before initial."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("invalid parameter."); - } - if (nullptr == _s_charsets[_index].charset) { - *eos = true; - return Status::OK(); - } - *eos = false; - return fill_one_row(tuple, pool); -} - Status SchemaCharsetsScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("call this before initial."); @@ -127,13 +66,12 @@ Status SchemaCharsetsScanner::_fill_block_impl(vectorized::Block* block) { // DEFAULT_COLLATE_NAME for (int i = 0; i < row_num; ++i) { StringRef str = StringRef(_s_charsets[i].default_collation, - strlen(_s_charsets[i].default_collation)); + strlen(_s_charsets[i].default_collation)); fill_dest_column(block, &str, _tuple_desc->slots()[1]); } // DESCRIPTION for (int i = 0; i < row_num; ++i) { - StringRef str = - StringRef(_s_charsets[i].description, strlen(_s_charsets[i].description)); + StringRef str = StringRef(_s_charsets[i].description, strlen(_s_charsets[i].description)); fill_dest_column(block, &str, _tuple_desc->slots()[2]); } // maxlen diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.h b/be/src/exec/schema_scanner/schema_charsets_scanner.h index 2b2043926578f4..765262cfecc6c3 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.h @@ -28,7 +28,6 @@ class SchemaCharsetsScanner : public SchemaScanner { SchemaCharsetsScanner(); virtual ~SchemaCharsetsScanner(); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); Status get_next_block(vectorized::Block* block, bool* eos) override; private: @@ -39,10 +38,8 @@ class SchemaCharsetsScanner : public SchemaScanner { int64_t maxlen; }; - Status fill_one_row(Tuple* tuple, MemPool* pool); Status _fill_block_impl(vectorized::Block* block); - int _index; static SchemaScanner::ColumnDesc _s_css_columns[]; static CharsetStruct _s_charsets[]; }; diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.cpp b/be/src/exec/schema_scanner/schema_collations_scanner.cpp index 0d2f5479becf78..1f7ef6f12d7cda 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_collations_scanner.cpp @@ -41,90 +41,10 @@ SchemaCollationsScanner::CollationStruct SchemaCollationsScanner::_s_collations[ SchemaCollationsScanner::SchemaCollationsScanner() : SchemaScanner(_s_cols_columns, sizeof(_s_cols_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_COLLATIONS), - _index(0) {} + TSchemaTableType::SCH_COLLATIONS) {} SchemaCollationsScanner::~SchemaCollationsScanner() {} -Status SchemaCollationsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // COLLATION_NAME - { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_collations[_index].name); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_collations[_index].name, len + 1); - str_slot->size = len; - } - // charset - { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_collations[_index].charset); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_collations[_index].charset, len + 1); - str_slot->size = len; - } - // id - { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - *(int64_t*)slot = _s_collations[_index].id; - } - // is_default - { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_collations[_index].is_default); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_collations[_index].is_default, len + 1); - str_slot->size = len; - } - // IS_COMPILED - { - void* slot = tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_collations[_index].is_compile); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_collations[_index].is_compile, len + 1); - str_slot->size = len; - } - // sortlen - { - void* slot = tuple->get_slot(_tuple_desc->slots()[5]->tuple_offset()); - *(int64_t*)slot = _s_collations[_index].sortlen; - } - _index++; - return Status::OK(); -} - -Status SchemaCollationsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("call this before initial."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("invalid parameter."); - } - if (nullptr == _s_collations[_index].name) { - *eos = true; - return Status::OK(); - } - - *eos = false; - return fill_one_row(tuple, pool); -} - Status SchemaCollationsScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("call this before initial."); @@ -144,34 +64,34 @@ Status SchemaCollationsScanner::_fill_block_impl(vectorized::Block* block) { } // COLLATION_NAME for (int i = 0; i < row_num; ++i) { - StringValue str = StringValue(_s_collations[i].name, strlen(_s_collations[i].name)); + StringRef str = StringRef(_s_collations[i].name, strlen(_s_collations[i].name)); fill_dest_column(block, &str, _tuple_desc->slots()[0]); } // charset for (int i = 0; i < row_num; ++i) { - StringValue str = StringValue(_s_collations[i].charset, strlen(_s_collations[i].charset)); + StringRef str = StringRef(_s_collations[i].charset, strlen(_s_collations[i].charset)); fill_dest_column(block, &str, _tuple_desc->slots()[1]); } // id for (int i = 0; i < row_num; ++i) { - int64_t src = _s_collations[_index].id; + int64_t src = _s_collations[i].id; fill_dest_column(block, &src, _tuple_desc->slots()[2]); } // is_default for (int i = 0; i < row_num; ++i) { - StringValue str = - StringValue(_s_collations[i].is_default, strlen(_s_collations[i].is_default)); + StringRef str = + StringRef(_s_collations[i].is_default, strlen(_s_collations[i].is_default)); fill_dest_column(block, &str, _tuple_desc->slots()[3]); } // IS_COMPILED for (int i = 0; i < row_num; ++i) { - StringValue str = - StringValue(_s_collations[i].is_compile, strlen(_s_collations[i].is_compile)); + StringRef str = + StringRef(_s_collations[i].is_compile, strlen(_s_collations[i].is_compile)); fill_dest_column(block, &str, _tuple_desc->slots()[4]); } // sortlen for (int i = 0; i < row_num; ++i) { - int64_t src = _s_collations[_index].sortlen; + int64_t src = _s_collations[i].sortlen; fill_dest_column(block, &src, _tuple_desc->slots()[5]); } return Status::OK(); diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.h b/be/src/exec/schema_scanner/schema_collations_scanner.h index 46883001e119d6..ce81446aca0371 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.h +++ b/be/src/exec/schema_scanner/schema_collations_scanner.h @@ -28,7 +28,6 @@ class SchemaCollationsScanner : public SchemaScanner { SchemaCollationsScanner(); virtual ~SchemaCollationsScanner(); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); Status get_next_block(vectorized::Block* block, bool* eos) override; private: @@ -41,10 +40,8 @@ class SchemaCollationsScanner : public SchemaScanner { int64_t sortlen; }; - Status fill_one_row(Tuple* tuple, MemPool* pool); Status _fill_block_impl(vectorized::Block* block); - int _index; static SchemaScanner::ColumnDesc _s_cols_columns[]; static CollationStruct _s_collations[]; }; diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp index c614e87cff7d20..eeaf4110f9ee5b 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp @@ -17,6 +17,8 @@ #include "exec/schema_scanner/schema_columns_scanner.h" +#include +#include #include #include "exec/schema_scanner/schema_helper.h" @@ -57,8 +59,7 @@ SchemaColumnsScanner::SchemaColumnsScanner() : SchemaScanner(_s_col_columns, sizeof(_s_col_columns) / sizeof(SchemaScanner::ColumnDesc), TSchemaTableType::SCH_COLUMNS), _db_index(0), - _table_index(0), - _column_index(0) {} + _table_index(0) {} SchemaColumnsScanner::~SchemaColumnsScanner() = default; @@ -96,7 +97,7 @@ Status SchemaColumnsScanner::start(RuntimeState* state) { } //For compatibility with mysql the result of DATA_TYPE in information_schema.columns -std::string SchemaColumnsScanner::to_mysql_data_type_string(TColumnDesc& desc) { +std::string SchemaColumnsScanner::_to_mysql_data_type_string(TColumnDesc& desc) { switch (desc.columnType) { case TPrimitiveType::BOOLEAN: return "tinyint"; @@ -137,7 +138,7 @@ std::string SchemaColumnsScanner::to_mysql_data_type_string(TColumnDesc& desc) { } } -std::string SchemaColumnsScanner::type_to_string(TColumnDesc& desc) { +std::string SchemaColumnsScanner::_type_to_string(TColumnDesc& desc) { switch (desc.columnType) { case TPrimitiveType::BOOLEAN: return "tinyint(1)"; @@ -219,239 +220,7 @@ std::string SchemaColumnsScanner::type_to_string(TColumnDesc& desc) { } } -// TODO: ALL schema algorithm like these need to be refactor to avoid UB and -// keep StringRef semantic to be kept. - -//fill row in the "INFORMATION_SCHEMA COLUMNS" -//Reference from https://dev.mysql.com/doc/refman/8.0/en/information-schema-columns-table.html -Status SchemaColumnsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - - // TABLE_CATALOG - { - if (!_db_result.__isset.catalogs) { - tuple->set_null(_tuple_desc->slots()[0]->null_indicator_offset()); - } else { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - // todo: we may change all of StringRef in similar usage - // to Slice someday to distinguish different purposes of use. - // or just merge them. - std::string catalog_name = _db_result.catalogs[_db_index - 1]; - str_slot->data = (char*)pool->allocate(catalog_name.size()); - str_slot->size = catalog_name.size(); - memcpy(const_cast(str_slot->data), catalog_name.c_str(), str_slot->size); - } - } - // TABLE_SCHEMA - { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); - str_slot->data = (char*)pool->allocate(db_name.size()); - str_slot->size = db_name.size(); - memcpy(const_cast(str_slot->data), db_name.c_str(), str_slot->size); - } - // TABLE_NAME - { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->data = (char*)pool->allocate(_table_result.tables[_table_index - 1].length()); - str_slot->size = _table_result.tables[_table_index - 1].length(); - memcpy(const_cast(str_slot->data), _table_result.tables[_table_index - 1].c_str(), - str_slot->size); - } - // COLUMN_NAME - { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->data = (char*)pool->allocate( - _desc_result.columns[_column_index].columnDesc.columnName.length()); - str_slot->size = _desc_result.columns[_column_index].columnDesc.columnName.length(); - memcpy(const_cast(str_slot->data), - _desc_result.columns[_column_index].columnDesc.columnName.c_str(), str_slot->size); - } - // ORDINAL_POSITION - { - void* slot = tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset()); - int64_t* bigint_slot = reinterpret_cast(slot); - *bigint_slot = _column_index + 1; - } - // COLUMN_DEFAULT - { tuple->set_null(_tuple_desc->slots()[5]->null_indicator_offset()); } - // IS_NULLABLE - { - void* slot = tuple->get_slot(_tuple_desc->slots()[6]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - - if (_desc_result.columns[_column_index].columnDesc.__isset.isAllowNull) { - if (_desc_result.columns[_column_index].columnDesc.isAllowNull) { - str_slot->size = strlen("YES"); - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "YES", str_slot->size); - } else { - str_slot->size = strlen("NO"); - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "NO", str_slot->size); - } - } else { - str_slot->size = strlen("NO"); - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "NO", str_slot->size); - } - } - // DATA_TYPE - { - void* slot = tuple->get_slot(_tuple_desc->slots()[7]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string buffer = - to_mysql_data_type_string(_desc_result.columns[_column_index].columnDesc); - str_slot->size = buffer.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), buffer.c_str(), str_slot->size); - } - // CHARACTER_MAXIMUM_LENGTH - // For string columns, the maximum length in characters. - { - int data_type = _desc_result.columns[_column_index].columnDesc.columnType; - if (data_type == TPrimitiveType::VARCHAR || data_type == TPrimitiveType::CHAR || - data_type == TPrimitiveType::STRING) { - void* slot = tuple->get_slot(_tuple_desc->slots()[8]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnLength) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnLength; - } else { - tuple->set_null(_tuple_desc->slots()[8]->null_indicator_offset()); - } - } else { - tuple->set_null(_tuple_desc->slots()[8]->null_indicator_offset()); - } - } - // CHARACTER_OCTET_LENGTH - // For string columns, the maximum length in bytes. - { - int data_type = _desc_result.columns[_column_index].columnDesc.columnType; - if (data_type == TPrimitiveType::VARCHAR || data_type == TPrimitiveType::CHAR || - data_type == TPrimitiveType::STRING) { - void* slot = tuple->get_slot(_tuple_desc->slots()[9]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnLength) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnLength * 4; - } else { - tuple->set_null(_tuple_desc->slots()[9]->null_indicator_offset()); - } - } else { - tuple->set_null(_tuple_desc->slots()[9]->null_indicator_offset()); - } - } - // NUMERIC_PRECISION - { - void* slot = tuple->get_slot(_tuple_desc->slots()[10]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnPrecision) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnPrecision; - } else { - tuple->set_null(_tuple_desc->slots()[10]->null_indicator_offset()); - } - } - // NUMERIC_SCALE - { - void* slot = tuple->get_slot(_tuple_desc->slots()[11]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnScale) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnScale; - } else { - tuple->set_null(_tuple_desc->slots()[11]->null_indicator_offset()); - } - } - // DATETIME_PRECISION - { tuple->set_null(_tuple_desc->slots()[12]->null_indicator_offset()); } - // CHARACTER_SET_NAME - { tuple->set_null(_tuple_desc->slots()[13]->null_indicator_offset()); } - // COLLATION_NAME - { tuple->set_null(_tuple_desc->slots()[14]->null_indicator_offset()); } - // COLUMN_TYPE - { - void* slot = tuple->get_slot(_tuple_desc->slots()[15]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string buffer = type_to_string(_desc_result.columns[_column_index].columnDesc); - str_slot->size = buffer.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), buffer.c_str(), str_slot->size); - } - // COLUMN_KEY - { - void* slot = tuple->get_slot(_tuple_desc->slots()[16]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnKey) { - str_slot->size = _desc_result.columns[_column_index].columnDesc.columnKey.length(); - str_slot->data = (char*)pool->allocate( - _desc_result.columns[_column_index].columnDesc.columnKey.length()); - memcpy(const_cast(str_slot->data), - _desc_result.columns[_column_index].columnDesc.columnKey.c_str(), - str_slot->size); - } else { - str_slot->size = strlen("") + 1; - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "", str_slot->size); - } - } - // EXTRA - { - void* slot = tuple->get_slot(_tuple_desc->slots()[17]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = strlen("") + 1; - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "", str_slot->size); - } - // PRIVILEGES - { - void* slot = tuple->get_slot(_tuple_desc->slots()[18]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = strlen("") + 1; - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "", str_slot->size); - } - // COLUMN_COMMENT - { - void* slot = tuple->get_slot(_tuple_desc->slots()[19]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->data = - (char*)pool->allocate(_desc_result.columns[_column_index].comment.length()); - str_slot->size = _desc_result.columns[_column_index].comment.length(); - memcpy(const_cast(str_slot->data), - _desc_result.columns[_column_index].comment.c_str(), str_slot->size); - } - // COLUMN_SIZE - { - void* slot = tuple->get_slot(_tuple_desc->slots()[20]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnLength) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnLength; - } else { - tuple->set_null(_tuple_desc->slots()[20]->null_indicator_offset()); - } - } - // DECIMAL_DIGITS - { - void* slot = tuple->get_slot(_tuple_desc->slots()[21]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnScale) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnScale; - } else { - tuple->set_null(_tuple_desc->slots()[21]->null_indicator_offset()); - } - } - // GENERATION_EXPRESSION - { tuple->set_null(_tuple_desc->slots()[22]->null_indicator_offset()); } - // SRS_ID - { tuple->set_null(_tuple_desc->slots()[23]->null_indicator_offset()); } - _column_index++; - return Status::OK(); -} - -Status SchemaColumnsScanner::get_new_desc() { +Status SchemaColumnsScanner::_get_new_desc() { TDescribeTableParams desc_params; desc_params.__set_db(_db_result.dbs[_db_index - 1]); if (_db_result.__isset.catalogs) { @@ -475,12 +244,11 @@ Status SchemaColumnsScanner::get_new_desc() { } else { return Status::InternalError("IP or port doesn't exists"); } - _column_index = 0; return Status::OK(); } -Status SchemaColumnsScanner::get_new_table() { +Status SchemaColumnsScanner::_get_new_table() { TGetTablesParams table_params; table_params.__set_db(_db_result.dbs[_db_index]); if (_db_result.__isset.catalogs) { @@ -511,30 +279,6 @@ Status SchemaColumnsScanner::get_new_table() { return Status::OK(); } -Status SchemaColumnsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("use this class before inited."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input parameter is nullptr."); - } - while (_column_index >= _desc_result.columns.size()) { - if (_table_index >= _table_result.tables.size()) { - if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); - } else { - *eos = true; - return Status::OK(); - } - } else { - RETURN_IF_ERROR(get_new_desc()); - } - } - - *eos = false; - return fill_one_row(tuple, pool); -} - Status SchemaColumnsScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("use this class before inited."); @@ -545,13 +289,13 @@ Status SchemaColumnsScanner::get_next_block(vectorized::Block* block, bool* eos) while (_table_index >= _table_result.tables.size()) { if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); } else { *eos = true; return Status::OK(); } } - RETURN_IF_ERROR(get_new_desc()); + RETURN_IF_ERROR(_get_new_desc()); *eos = false; return _fill_block_impl(block); @@ -568,7 +312,7 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { } } else { std::string catalog_name = _db_result.catalogs[_db_index - 1]; - StringValue str = StringValue(catalog_name.c_str(), catalog_name.size()); + StringRef str = StringRef(catalog_name.c_str(), catalog_name.size()); for (int i = 0; i < columns_num; ++i) { fill_dest_column(block, &str, _tuple_desc->slots()[0]); } @@ -577,15 +321,15 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { // TABLE_SCHEMA { std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); - StringValue str = StringValue(db_name.c_str(), db_name.size()); + StringRef str = StringRef(db_name.c_str(), db_name.size()); for (int i = 0; i < columns_num; ++i) { fill_dest_column(block, &str, _tuple_desc->slots()[1]); } } // TABLE_NAME { - StringValue str = StringValue(_table_result.tables[_table_index - 1].c_str(), - _table_result.tables[_table_index - 1].length()); + StringRef str = StringRef(_table_result.tables[_table_index - 1].c_str(), + _table_result.tables[_table_index - 1].length()); for (int i = 0; i < columns_num; ++i) { fill_dest_column(block, &str, _tuple_desc->slots()[2]); } @@ -593,8 +337,8 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { // COLUMN_NAME { for (int i = 0; i < columns_num; ++i) { - StringValue str = StringValue(_desc_result.columns[i].columnDesc.columnName.c_str(), - _desc_result.columns[i].columnDesc.columnName.length()); + StringRef str = StringRef(_desc_result.columns[i].columnDesc.columnName.c_str(), + _desc_result.columns[i].columnDesc.columnName.length()); fill_dest_column(block, &str, _tuple_desc->slots()[3]); } } @@ -616,14 +360,14 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < columns_num; ++i) { if (_desc_result.columns[i].columnDesc.__isset.isAllowNull) { if (_desc_result.columns[i].columnDesc.isAllowNull) { - StringValue str = StringValue("YES", 3); + StringRef str = StringRef("YES", 3); fill_dest_column(block, &str, _tuple_desc->slots()[6]); } else { - StringValue str = StringValue("NO", 2); + StringRef str = StringRef("NO", 2); fill_dest_column(block, &str, _tuple_desc->slots()[6]); } } else { - StringValue str = StringValue("NO", 2); + StringRef str = StringRef("NO", 2); fill_dest_column(block, &str, _tuple_desc->slots()[6]); } } @@ -631,8 +375,8 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { // DATA_TYPE { for (int i = 0; i < columns_num; ++i) { - std::string buffer = to_mysql_data_type_string(_desc_result.columns[i].columnDesc); - StringValue str = StringValue(buffer.c_str(), buffer.length()); + std::string buffer = _to_mysql_data_type_string(_desc_result.columns[i].columnDesc); + StringRef str = StringRef(buffer.c_str(), buffer.length()); fill_dest_column(block, &str, _tuple_desc->slots()[7]); } } @@ -715,8 +459,8 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { // COLUMN_TYPE { for (int i = 0; i < columns_num; ++i) { - std::string buffer = type_to_string(_desc_result.columns[i].columnDesc); - StringValue str = StringValue(buffer.c_str(), buffer.length()); + std::string buffer = _type_to_string(_desc_result.columns[i].columnDesc); + StringRef str = StringRef(buffer.c_str(), buffer.length()); fill_dest_column(block, &str, _tuple_desc->slots()[15]); } } @@ -724,12 +468,11 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { { for (int i = 0; i < columns_num; ++i) { if (_desc_result.columns[i].columnDesc.__isset.columnKey) { - StringValue str = - StringValue(_desc_result.columns[i].columnDesc.columnKey.c_str(), - _desc_result.columns[i].columnDesc.columnKey.length()); + StringRef str = StringRef(_desc_result.columns[i].columnDesc.columnKey.c_str(), + _desc_result.columns[i].columnDesc.columnKey.length()); fill_dest_column(block, &str, _tuple_desc->slots()[16]); } else { - StringValue str = StringValue("", 0); + StringRef str = StringRef("", 0); fill_dest_column(block, &str, _tuple_desc->slots()[16]); } } @@ -737,22 +480,22 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { // EXTRA { for (int i = 0; i < columns_num; ++i) { - StringValue str = StringValue("", 0); + StringRef str = StringRef("", 0); fill_dest_column(block, &str, _tuple_desc->slots()[17]); } } // PRIVILEGES { for (int i = 0; i < columns_num; ++i) { - StringValue str = StringValue("", 0); + StringRef str = StringRef("", 0); fill_dest_column(block, &str, _tuple_desc->slots()[18]); } } // COLUMN_COMMENT { for (int i = 0; i < columns_num; ++i) { - StringValue str = StringValue(_desc_result.columns[i].comment.c_str(), - _desc_result.columns[i].comment.length()); + StringRef str = StringRef(_desc_result.columns[i].comment.c_str(), + _desc_result.columns[i].comment.length()); fill_dest_column(block, &str, _tuple_desc->slots()[19]); } } diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.h b/be/src/exec/schema_scanner/schema_columns_scanner.h index adc89ff7452b81..2a14db79a4357c 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.h +++ b/be/src/exec/schema_scanner/schema_columns_scanner.h @@ -29,21 +29,18 @@ class SchemaColumnsScanner : public SchemaScanner { SchemaColumnsScanner(); virtual ~SchemaColumnsScanner(); virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _get_new_table(); + Status _get_new_desc(); + Status _get_create_table(std::string* result); Status _fill_block_impl(vectorized::Block* block); - Status get_new_desc(); - Status get_create_table(std::string* result); - std::string to_mysql_data_type_string(TColumnDesc& desc); - std::string type_to_string(TColumnDesc& desc); + std::string _to_mysql_data_type_string(TColumnDesc& desc); + std::string _type_to_string(TColumnDesc& desc); int _db_index; int _table_index; - int _column_index; TGetDbsResult _db_result; TGetTablesResult _table_result; TDescribeTableResult _desc_result; diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp index 665947fdd839ef..f195d0dfc75635 100644 --- a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp @@ -31,7 +31,7 @@ Status SchemaDummyScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaDummyScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaDummyScanner::get_next_block(vectorized::Block* block, bool* eos) { *eos = true; return Status::OK(); } diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.h b/be/src/exec/schema_scanner/schema_dummy_scanner.h index 2b661ae58bad24..1bef2e547f1488 100644 --- a/be/src/exec/schema_scanner/schema_dummy_scanner.h +++ b/be/src/exec/schema_scanner/schema_dummy_scanner.h @@ -26,7 +26,7 @@ class SchemaDummyScanner : public SchemaScanner { SchemaDummyScanner(); virtual ~SchemaDummyScanner(); virtual Status start(RuntimeState* state = nullptr); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_files_scanner.cpp b/be/src/exec/schema_scanner/schema_files_scanner.cpp index c9a6b9721d700b..a0cda249dac01d 100644 --- a/be/src/exec/schema_scanner/schema_files_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_files_scanner.cpp @@ -67,7 +67,8 @@ SchemaScanner::ColumnDesc SchemaFilesScanner::_s_tbls_columns[] = { SchemaFilesScanner::SchemaFilesScanner() : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), + sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_FILES), _db_index(0), _table_index(0) {} @@ -78,24 +79,24 @@ Status SchemaFilesScanner::start(RuntimeState* state) { return Status::InternalError("used before initialized."); } TGetDbsParams db_params; - if (NULL != _param->db) { + if (nullptr != _param->db) { db_params.__set_pattern(*(_param->db)); } if (nullptr != _param->catalog) { db_params.__set_catalog(*(_param->catalog)); } - if (NULL != _param->current_user_ident) { + if (nullptr != _param->current_user_ident) { db_params.__set_current_user_ident(*(_param->current_user_ident)); } else { - if (NULL != _param->user) { + if (nullptr != _param->user) { db_params.__set_user(*(_param->user)); } - if (NULL != _param->user_ip) { + if (nullptr != _param->user_ip) { db_params.__set_user_ip(*(_param->user_ip)); } } - if (NULL != _param->ip && 0 != _param->port) { + if (nullptr != _param->ip && 0 != _param->port) { RETURN_IF_ERROR( SchemaHelper::get_db_names(*(_param->ip), _param->port, db_params, &_db_result)); } else { @@ -104,11 +105,11 @@ Status SchemaFilesScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaFilesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaFilesScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } *eos = true; diff --git a/be/src/exec/schema_scanner/schema_files_scanner.h b/be/src/exec/schema_scanner/schema_files_scanner.h index 012d1ed097c6a1..831a748cf70528 100644 --- a/be/src/exec/schema_scanner/schema_files_scanner.h +++ b/be/src/exec/schema_scanner/schema_files_scanner.h @@ -28,7 +28,7 @@ class SchemaFilesScanner : public SchemaScanner { virtual ~SchemaFilesScanner(); virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; int _db_index; int _table_index; diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp index 71c10ce1939b3d..6b268e6563fe2f 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp @@ -55,7 +55,8 @@ SchemaScanner::ColumnDesc SchemaPartitionsScanner::_s_tbls_columns[] = { SchemaPartitionsScanner::SchemaPartitionsScanner() : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), + sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_PARTITIONS), _db_index(0), _table_index(0) {} @@ -66,24 +67,24 @@ Status SchemaPartitionsScanner::start(RuntimeState* state) { return Status::InternalError("used before initialized."); } TGetDbsParams db_params; - if (NULL != _param->db) { + if (nullptr != _param->db) { db_params.__set_pattern(*(_param->db)); } if (nullptr != _param->catalog) { db_params.__set_catalog(*(_param->catalog)); } - if (NULL != _param->current_user_ident) { + if (nullptr != _param->current_user_ident) { db_params.__set_current_user_ident(*(_param->current_user_ident)); } else { - if (NULL != _param->user) { + if (nullptr != _param->user) { db_params.__set_user(*(_param->user)); } - if (NULL != _param->user_ip) { + if (nullptr != _param->user_ip) { db_params.__set_user_ip(*(_param->user_ip)); } } - if (NULL != _param->ip && 0 != _param->port) { + if (nullptr != _param->ip && 0 != _param->port) { RETURN_IF_ERROR( SchemaHelper::get_db_names(*(_param->ip), _param->port, db_params, &_db_result)); } else { @@ -92,11 +93,11 @@ Status SchemaPartitionsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaPartitionsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaPartitionsScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } *eos = true; diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.h b/be/src/exec/schema_scanner/schema_partitions_scanner.h index 5bce10cfbbaa17..fc8490afb173cb 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.h +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.h @@ -28,7 +28,7 @@ class SchemaPartitionsScanner : public SchemaScanner { virtual ~SchemaPartitionsScanner(); virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; int _db_index; int _table_index; diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp index 800be0c1aad2a6..599b8e0d8fffa6 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp @@ -51,35 +51,20 @@ SchemaScanner::ColumnDesc SchemaRowsetsScanner::_s_tbls_columns[] = { SchemaRowsetsScanner::SchemaRowsetsScanner() : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - backend_id_(0), - rowsets_idx_(0) {}; + sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_ROWSETS), + backend_id_(0) {}; Status SchemaRowsetsScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } backend_id_ = state->backend_id(); - RETURN_IF_ERROR(get_all_rowsets()); + RETURN_IF_ERROR(_get_all_rowsets()); return Status::OK(); } -Status SchemaRowsetsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("Used before initialized."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input pointer is nullptr."); - } - if (rowsets_idx_ >= rowsets_.size()) { - *eos = true; - return Status::OK(); - } - *eos = false; - return fill_one_row(tuple, pool); -} - -Status SchemaRowsetsScanner::get_all_rowsets() { +Status SchemaRowsetsScanner::_get_all_rowsets() { std::vector tablets = StorageEngine::instance()->tablet_manager()->get_all_tablet(); for (const auto& tablet : tablets) { @@ -97,80 +82,127 @@ Status SchemaRowsetsScanner::get_all_rowsets() { return Status::OK(); } -Status SchemaRowsetsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - RowsetSharedPtr rowset = rowsets_[rowsets_idx_]; +Status SchemaRowsetsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + *eos = true; + if (!rowsets_.size()) { + return Status::OK(); + } + return _fill_block_impl(block); +} + +Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { + auto rowsets_num = rowsets_.size(); // BACKEND_ID { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - *(reinterpret_cast(slot)) = backend_id_; + int64_t src = backend_id_; + for (int i = 0; i < rowsets_num; ++i) { + fill_dest_column(block, &src, _tuple_desc->slots()[0]); + } } // ROWSET_ID { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string rowset_id = rowset->rowset_id().to_string(); - str_slot->data = (char*)pool->allocate(rowset_id.size()); - str_slot->size = rowset_id.size(); - memcpy(const_cast(str_slot->data), rowset_id.c_str(), str_slot->size); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + std::string rowset_id = rowset->rowset_id().to_string(); + StringRef str = StringRef(rowset_id.c_str(), rowset_id.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[1]); + } } // TABLET_ID { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->rowset_meta()->tablet_id(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->rowset_meta()->tablet_id(); + fill_dest_column(block, &src, _tuple_desc->slots()[2]); + } } // ROWSET_NUM_ROWS { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->num_rows(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->num_rows(); + fill_dest_column(block, &src, _tuple_desc->slots()[3]); + } } // TXN_ID { - void* slot = tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->txn_id(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->txn_id(); + fill_dest_column(block, &src, _tuple_desc->slots()[4]); + } } // NUM_SEGMENTS { - void* slot = tuple->get_slot(_tuple_desc->slots()[5]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->num_segments(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->num_segments(); + fill_dest_column(block, &src, _tuple_desc->slots()[5]); + } } // START_VERSION { - void* slot = tuple->get_slot(_tuple_desc->slots()[6]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->start_version(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->start_version(); + fill_dest_column(block, &src, _tuple_desc->slots()[6]); + } } // END_VERSION { - void* slot = tuple->get_slot(_tuple_desc->slots()[7]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->end_version(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->end_version(); + fill_dest_column(block, &src, _tuple_desc->slots()[7]); + } } // INDEX_DISK_SIZE { - void* slot = tuple->get_slot(_tuple_desc->slots()[8]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->index_disk_size(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + size_t src = rowset->index_disk_size(); + fill_dest_column(block, &src, _tuple_desc->slots()[8]); + } } // DATA_DISK_SIZE { - void* slot = tuple->get_slot(_tuple_desc->slots()[9]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->data_disk_size(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + size_t src = rowset->data_disk_size(); + fill_dest_column(block, &src, _tuple_desc->slots()[9]); + } } // CREATION_TIME { - void* slot = tuple->get_slot(_tuple_desc->slots()[10]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->creation_time(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + size_t src = rowset->creation_time(); + fill_dest_column(block, &src, _tuple_desc->slots()[10]); + } } // OLDEST_WRITE_TIMESTAMP { - void* slot = tuple->get_slot(_tuple_desc->slots()[11]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->oldest_write_timestamp(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + size_t src = rowset->oldest_write_timestamp(); + fill_dest_column(block, &src, _tuple_desc->slots()[11]); + } } // NEWEST_WRITE_TIMESTAMP { - void* slot = tuple->get_slot(_tuple_desc->slots()[12]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->newest_write_timestamp(); + for (int i = 0; i < rowsets_num; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + size_t src = rowset->newest_write_timestamp(); + fill_dest_column(block, &src, _tuple_desc->slots()[12]); + } } - ++rowsets_idx_; return Status::OK(); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.h b/be/src/exec/schema_scanner/schema_rowsets_scanner.h index dd642225fad51b..92341b7197be41 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.h @@ -33,18 +33,14 @@ class SchemaRowsetsScanner : public SchemaScanner { ~SchemaRowsetsScanner() override = default; Status start(RuntimeState* state) override; - Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_all_rowsets(); - // Status get_new_segments(); - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _get_all_rowsets(); + Status _fill_block_impl(vectorized::Block* block); -private: static SchemaScanner::ColumnDesc _s_tbls_columns[]; int64_t backend_id_ = 0; std::vector rowsets_; - // used for traversing rowsets_ - int rowsets_idx_ = 0; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp index 6b082ba82af755..cf70c414c45d8a 100644 --- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp @@ -34,8 +34,8 @@ SchemaScanner::ColumnDesc SchemaSchemaPrivilegesScanner::_s_tbls_columns[] = { SchemaSchemaPrivilegesScanner::SchemaSchemaPrivilegesScanner() : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _priv_index(0) {} + sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_SCHEMA_PRIVILEGES) {} SchemaSchemaPrivilegesScanner::~SchemaSchemaPrivilegesScanner() {} @@ -43,76 +43,11 @@ Status SchemaSchemaPrivilegesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); return Status::OK(); } -Status SchemaSchemaPrivilegesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - const TPrivilegeStatus& priv_status = _priv_result.privileges[_priv_index]; - // grantee - { - Status status = fill_one_col(&priv_status.grantee, pool, - tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // catalog - // This value is always def. - { - std::string definer = "def"; - Status status = fill_one_col(&definer, pool, - tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // schema - { - Status status = fill_one_col(&priv_status.schema, pool, - tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // privilege type - { - Status status = fill_one_col(&priv_status.privilege_type, pool, - tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // is grantable - { - Status status = fill_one_col(&priv_status.is_grantable, pool, - tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - _priv_index++; - return Status::OK(); -} - -Status SchemaSchemaPrivilegesScanner::fill_one_col(const std::string* src, MemPool* pool, - void* slot) { - if (nullptr == slot || nullptr == pool || nullptr == src) { - return Status::InternalError("input pointer is nullptr."); - } - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - return Status::OK(); -} - -Status SchemaSchemaPrivilegesScanner::get_new_table() { +Status SchemaSchemaPrivilegesScanner::_get_new_table() { TGetTablesParams table_params; if (nullptr != _param->wild) { table_params.__set_pattern(*(_param->wild)); @@ -134,23 +69,71 @@ Status SchemaSchemaPrivilegesScanner::get_new_table() { } else { return Status::InternalError("IP or port doesn't exists"); } - _priv_index = 0; return Status::OK(); } -Status SchemaSchemaPrivilegesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaSchemaPrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - if (_priv_index >= _priv_result.privileges.size()) { - *eos = true; + + *eos = true; + if (!_priv_result.privileges.size()) { return Status::OK(); } - *eos = false; - return fill_one_row(tuple, pool); + return _fill_block_impl(block); +} + +Status SchemaSchemaPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { + auto privileges_num = _priv_result.privileges.size(); + + // grantee + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[0]); + } + } + // catalog + // This value is always def. + { + std::string definer = "def"; + StringRef str = StringRef(definer.c_str(), definer.size()); + for (int i = 0; i < privileges_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[1]); + } + } + // schema + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.schema.c_str(), priv_status.schema.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[2]); + } + } + // privilege type + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.privilege_type.c_str(), + priv_status.privilege_type.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[3]); + } + } + // is grantable + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = + StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[4]); + } + } + return Status::OK(); } } // namespace doris \ No newline at end of file diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h index d2f7c0f9b6cd42..0782983f76b023 100644 --- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h @@ -28,14 +28,12 @@ class SchemaSchemaPrivilegesScanner : public SchemaScanner { virtual ~SchemaSchemaPrivilegesScanner(); virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); - Status fill_one_col(const std::string* src, MemPool* pool, void* slot); + Status _get_new_table(); + Status _fill_block_impl(vectorized::Block* block); - int _priv_index; TListPrivilegesResult _priv_result; static SchemaScanner::ColumnDesc _s_tbls_columns[]; }; diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp index 10427963f9089b..d4ef60c2b69a78 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp @@ -33,8 +33,8 @@ SchemaScanner::ColumnDesc SchemaSchemataScanner::_s_columns[] = { }; SchemaSchemataScanner::SchemaSchemataScanner() - : SchemaScanner(_s_columns, sizeof(_s_columns) / sizeof(SchemaScanner::ColumnDesc)), - _db_index(0) {} + : SchemaScanner(_s_columns, sizeof(_s_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_SCHEMATA) {} SchemaSchemataScanner::~SchemaSchemataScanner() = default; @@ -70,73 +70,67 @@ Status SchemaSchemataScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaSchemataScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); +Status SchemaSchemataScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before Initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + *eos = true; + if (!_db_result.dbs.size()) { + return Status::OK(); + } + return _fill_block_impl(block); +} + +Status SchemaSchemataScanner::_fill_block_impl(vectorized::Block* block) { + auto dbs_num = _db_result.dbs.size(); // catalog { - if (!_db_result.__isset.catalogs) { - tuple->set_null(_tuple_desc->slots()[0]->null_indicator_offset()); - } else { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string catalog_name = _db_result.catalogs[_db_index]; - str_slot->data = (char*)pool->allocate(catalog_name.size()); - str_slot->size = catalog_name.size(); - memcpy(const_cast(str_slot->data), catalog_name.c_str(), str_slot->size); + for (int i = 0; i < dbs_num; ++i) { + if (!_db_result.__isset.catalogs) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[0]); + } else { + std::string catalog_name = _db_result.catalogs[i]; + StringRef str = StringRef(catalog_name.c_str(), catalog_name.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[0]); + } } } // schema { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index]); - str_slot->data = (char*)pool->allocate(db_name.size()); - str_slot->size = db_name.size(); - memcpy(const_cast(str_slot->data), db_name.c_str(), str_slot->size); + for (int i = 0; i < dbs_num; ++i) { + std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[i]); + StringRef str = StringRef(db_name.c_str(), db_name.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[1]); + } } // DEFAULT_CHARACTER_SET_NAME { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = strlen("utf8") + 1; - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); + for (int i = 0; i < dbs_num; ++i) { + std::string src = "utf8"; + StringRef str = StringRef(src.c_str(), src.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[2]); } - memcpy(const_cast(str_slot->data), "utf8", str_slot->size); } // DEFAULT_COLLATION_NAME { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = strlen("utf8_general_ci") + 1; - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); + for (int i = 0; i < dbs_num; ++i) { + std::string src = "utf8_general_ci"; + StringRef str = StringRef(src.c_str(), src.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[3]); } - memcpy(const_cast(str_slot->data), "utf8_general_ci", str_slot->size); } // SQL_PATH - { tuple->set_null(_tuple_desc->slots()[4]->null_indicator_offset()); } - _db_index++; - return Status::OK(); -} - -Status SchemaSchemataScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("Used before Initialized."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input pointer is nullptr."); - } - if (_db_index >= _db_result.dbs.size()) { - *eos = true; - return Status::OK(); + { + for (int i = 0; i < dbs_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[4]); + } } - *eos = false; - return fill_one_row(tuple, pool); + return Status::OK(); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.h b/be/src/exec/schema_scanner/schema_schemata_scanner.h index f6bc752021aaf0..26d97b14c7ceef 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.h +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.h @@ -28,12 +28,11 @@ class SchemaSchemataScanner : public SchemaScanner { virtual ~SchemaSchemataScanner(); virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); - int _db_index; TGetDbsResult _db_result; static SchemaScanner::ColumnDesc _s_columns[]; }; diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp index 1d572bc27650d3..7290580cfa4fb2 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp @@ -17,6 +17,8 @@ #include "exec/schema_scanner/schema_table_privileges_scanner.h" +#include + #include "exec/schema_scanner/schema_helper.h" #include "runtime/primitive_type.h" #include "vec/common/string_ref.h" @@ -35,8 +37,8 @@ SchemaScanner::ColumnDesc SchemaTablePrivilegesScanner::_s_tbls_columns[] = { SchemaTablePrivilegesScanner::SchemaTablePrivilegesScanner() : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _priv_index(0) {} + sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_TABLE_PRIVILEGES) {} SchemaTablePrivilegesScanner::~SchemaTablePrivilegesScanner() {} @@ -44,84 +46,11 @@ Status SchemaTablePrivilegesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } - RETURN_IF_ERROR(get_new_table()); - return Status::OK(); -} - -Status SchemaTablePrivilegesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - const TPrivilegeStatus& priv_status = _priv_result.privileges[_priv_index]; - // grantee - { - Status status = fill_one_col(&priv_status.grantee, pool, - tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // catalog - // This value is always def. - { - std::string definer = "def"; - Status status = fill_one_col(&definer, pool, - tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // schema - { - Status status = fill_one_col(&priv_status.schema, pool, - tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // table name - { - Status status = fill_one_col(&priv_status.table_name, pool, - tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // privilege type - { - Status status = fill_one_col(&priv_status.privilege_type, pool, - tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // is grantable - { - Status status = fill_one_col(&priv_status.is_grantable, pool, - tuple->get_slot(_tuple_desc->slots()[5]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - _priv_index++; + RETURN_IF_ERROR(_get_new_table()); return Status::OK(); } -Status SchemaTablePrivilegesScanner::fill_one_col(const std::string* src, MemPool* pool, - void* slot) { - if (nullptr == slot || nullptr == pool || nullptr == src) { - return Status::InternalError("input pointer is nullptr."); - } - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - return Status::OK(); -} - -Status SchemaTablePrivilegesScanner::get_new_table() { +Status SchemaTablePrivilegesScanner::_get_new_table() { TGetTablesParams table_params; if (nullptr != _param->wild) { table_params.__set_pattern(*(_param->wild)); @@ -143,23 +72,80 @@ Status SchemaTablePrivilegesScanner::get_new_table() { } else { return Status::InternalError("IP or port doesn't exists"); } - _priv_index = 0; return Status::OK(); } -Status SchemaTablePrivilegesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaTablePrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - if (_priv_index >= _priv_result.privileges.size()) { - *eos = true; + + *eos = true; + if (!_priv_result.privileges.size()) { return Status::OK(); } - *eos = false; - return fill_one_row(tuple, pool); + return _fill_block_impl(block); +} + +Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block* block) { + auto privileges_num = _priv_result.privileges.size(); + + // grantee + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[0]); + } + } + // catalog + // This value is always def. + { + std::string definer = "def"; + StringRef str = StringRef(definer.c_str(), definer.size()); + for (int i = 0; i < privileges_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[1]); + } + } + // schema + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.schema.c_str(), priv_status.schema.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[2]); + } + } + // table name + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = + StringRef(priv_status.table_name.c_str(), priv_status.table_name.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[3]); + } + } + // privilege type + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.privilege_type.c_str(), + priv_status.privilege_type.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[4]); + } + } + // is grantable + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = + StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[5]); + } + } + return Status::OK(); } } // namespace doris \ No newline at end of file diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h index d14e8005220832..cd8f40257cfa1c 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h @@ -28,14 +28,12 @@ class SchemaTablePrivilegesScanner : public SchemaScanner { virtual ~SchemaTablePrivilegesScanner(); virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); - Status fill_one_col(const std::string* src, MemPool* pool, void* slot); + Status _get_new_table(); + Status _fill_block_impl(vectorized::Block* block); - int _priv_index; TListPrivilegesResult _priv_result; static SchemaScanner::ColumnDesc _s_tbls_columns[]; }; diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index a7115d847a256e..d9fd19f7e360c1 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -89,11 +89,7 @@ Status SchemaTablesScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaTablesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - return Status::OK(); -} - -Status SchemaTablesScanner::get_new_table() { +Status SchemaTablesScanner::_get_new_table() { TGetTablesParams table_params; table_params.__set_db(_db_result.dbs[_db_index]); if (_db_result.__isset.catalogs) { @@ -123,16 +119,6 @@ Status SchemaTablesScanner::get_new_table() { return Status::OK(); } -Status SchemaTablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("Used before initialized."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input pointer is nullptr."); - } - return fill_one_row(tuple, pool); -} - Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { auto table_num = _table_result.tables.size(); // catalog @@ -311,7 +297,7 @@ Status SchemaTablesScanner::get_next_block(vectorized::Block* block, bool* eos) return Status::InternalError("input pointer is nullptr."); } if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); } else { *eos = true; return Status::OK(); diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h index 58d69e75ffcefa..6b7f2a65098cd0 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_tables_scanner.h @@ -30,12 +30,10 @@ class SchemaTablesScanner : public SchemaScanner { virtual ~SchemaTablesScanner(); virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _get_new_table(); Status _fill_block_impl(vectorized::Block* block); int _db_index; diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp index e2a1b86eb23e52..4e2be50ca8de10 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp @@ -33,8 +33,8 @@ SchemaScanner::ColumnDesc SchemaUserPrivilegesScanner::_s_tbls_columns[] = { SchemaUserPrivilegesScanner::SchemaUserPrivilegesScanner() : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _priv_index(0) {} + sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_USER_PRIVILEGES) {} SchemaUserPrivilegesScanner::~SchemaUserPrivilegesScanner() {} @@ -42,68 +42,11 @@ Status SchemaUserPrivilegesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); return Status::OK(); } -Status SchemaUserPrivilegesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - const TPrivilegeStatus& priv_status = _priv_result.privileges[_priv_index]; - // grantee - { - Status status = fill_one_col(&priv_status.grantee, pool, - tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // catalog - // This value is always def. - { - std::string definer = "def"; - Status status = fill_one_col(&definer, pool, - tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // privilege type - { - Status status = fill_one_col(&priv_status.privilege_type, pool, - tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // is grantable - { - Status status = fill_one_col(&priv_status.is_grantable, pool, - tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - _priv_index++; - return Status::OK(); -} - -Status SchemaUserPrivilegesScanner::fill_one_col(const std::string* src, MemPool* pool, - void* slot) { - if (nullptr == slot || nullptr == pool || nullptr == src) { - return Status::InternalError("input pointer is nullptr."); - } - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - return Status::OK(); -} - -Status SchemaUserPrivilegesScanner::get_new_table() { +Status SchemaUserPrivilegesScanner::_get_new_table() { TGetTablesParams table_params; if (nullptr != _param->wild) { table_params.__set_pattern(*(_param->wild)); @@ -125,23 +68,63 @@ Status SchemaUserPrivilegesScanner::get_new_table() { } else { return Status::InternalError("IP or port doesn't exists"); } - _priv_index = 0; return Status::OK(); } -Status SchemaUserPrivilegesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaUserPrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - if (_priv_index >= _priv_result.privileges.size()) { - *eos = true; + + *eos = true; + if (!_priv_result.privileges.size()) { return Status::OK(); } - *eos = false; - return fill_one_row(tuple, pool); + return _fill_block_impl(block); +} + +Status SchemaUserPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { + auto privileges_num = _priv_result.privileges.size(); + + // grantee + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[0]); + } + } + // catalog + // This value is always def. + { + std::string definer = "def"; + StringRef str = StringRef(definer.c_str(), definer.size()); + for (int i = 0; i < privileges_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[1]); + } + } + // privilege type + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.privilege_type.c_str(), + priv_status.privilege_type.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[2]); + } + } + // is grantable + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = + StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[3]); + } + } + return Status::OK(); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h index 2e2cbc80dafb61..4ec35e4cfdf417 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h @@ -28,14 +28,12 @@ class SchemaUserPrivilegesScanner : public SchemaScanner { virtual ~SchemaUserPrivilegesScanner(); virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); - Status fill_one_col(const std::string* src, MemPool* pool, void* slot); + Status _get_new_table(); + Status _fill_block_impl(vectorized::Block* block); - int _priv_index; TListPrivilegesResult _priv_result; static SchemaScanner::ColumnDesc _s_tbls_columns[]; }; diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.cpp b/be/src/exec/schema_scanner/schema_variables_scanner.cpp index f550e8a0dd2123..9f84c0dd17fae9 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_variables_scanner.cpp @@ -32,7 +32,8 @@ SchemaScanner::ColumnDesc SchemaVariablesScanner::_s_vars_columns[] = { SchemaVariablesScanner::SchemaVariablesScanner(TVarType::type type) : SchemaScanner(_s_vars_columns, - sizeof(_s_vars_columns) / sizeof(SchemaScanner::ColumnDesc)), + sizeof(_s_vars_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_VARIABLES), _type(type) {} SchemaVariablesScanner::~SchemaVariablesScanner() {} @@ -57,52 +58,40 @@ Status SchemaVariablesScanner::start(RuntimeState* state) { } else { return Status::InternalError("IP or port doesn't exists"); } - _begin = _var_result.variables.begin(); return Status::OK(); } -Status SchemaVariablesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { +Status SchemaVariablesScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("call this before initial."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("invalid parameter."); + } + + *eos = true; + if (_var_result.variables.empty()) { + return Status::OK(); + } + return _fill_block_impl(block); +} + +Status SchemaVariablesScanner::_fill_block_impl(vectorized::Block* block) { // variables names { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_begin->first.c_str()); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); + for (auto& it : _var_result.variables) { + StringRef str = StringRef(it.first.c_str(), it.first.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[0]); } - memcpy(const_cast(str_slot->data), _begin->first.c_str(), len + 1); - str_slot->size = len; } // value { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_begin->second.c_str()); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); + for (auto& it : _var_result.variables) { + StringRef str = StringRef(it.second.c_str(), it.second.size()); + fill_dest_column(block, &str, _tuple_desc->slots()[1]); } - memcpy(const_cast(str_slot->data), _begin->second.c_str(), len + 1); - str_slot->size = len; } - ++_begin; return Status::OK(); } -Status SchemaVariablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("call this before initial."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("invalid parameter."); - } - if (_begin == _var_result.variables.end()) { - *eos = true; - return Status::OK(); - } - *eos = false; - return fill_one_row(tuple, pool); -} - } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.h b/be/src/exec/schema_scanner/schema_variables_scanner.h index 25daaf0e4f9c43..21111ae9fe7087 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.h +++ b/be/src/exec/schema_scanner/schema_variables_scanner.h @@ -31,7 +31,7 @@ class SchemaVariablesScanner : public SchemaScanner { virtual ~SchemaVariablesScanner(); virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: struct VariableStruct { @@ -39,13 +39,12 @@ class SchemaVariablesScanner : public SchemaScanner { const char* value; }; - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); static SchemaScanner::ColumnDesc _s_vars_columns[]; TShowVariableResult _var_result; TVarType::type _type; - std::map::iterator _begin; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_views_scanner.cpp b/be/src/exec/schema_scanner/schema_views_scanner.cpp index af04de20df6be3..568f871b6e9fdb 100644 --- a/be/src/exec/schema_scanner/schema_views_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_views_scanner.cpp @@ -39,9 +39,9 @@ SchemaScanner::ColumnDesc SchemaViewsScanner::_s_tbls_columns[] = { SchemaViewsScanner::SchemaViewsScanner() : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _db_index(0), - _table_index(0) {} + sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), + TSchemaTableType::SCH_VIEWS), + _db_index(0) {} SchemaViewsScanner::~SchemaViewsScanner() {} @@ -76,161 +76,136 @@ Status SchemaViewsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaViewsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - const TTableStatus& tbl_status = _table_result.tables[_table_index]; +Status SchemaViewsScanner::_get_new_table() { + TGetTablesParams table_params; + table_params.__set_db(_db_result.dbs[_db_index++]); + if (nullptr != _param->wild) { + table_params.__set_pattern(*(_param->wild)); + } + if (nullptr != _param->current_user_ident) { + table_params.__set_current_user_ident(*(_param->current_user_ident)); + } else { + if (nullptr != _param->user) { + table_params.__set_user(*(_param->user)); + } + if (nullptr != _param->user_ip) { + table_params.__set_user_ip(*(_param->user_ip)); + } + } + table_params.__set_type("VIEW"); + + if (nullptr != _param->ip && 0 != _param->port) { + RETURN_IF_ERROR(SchemaHelper::list_table_status(*(_param->ip), _param->port, table_params, + &_table_result)); + } else { + return Status::InternalError("IP or port doesn't exists"); + } + return Status::OK(); +} + +Status SchemaViewsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(_get_new_table()); + } else { + *eos = true; + return Status::OK(); + } + *eos = false; + return _fill_block_impl(block); +} + +Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { + auto tables_num = _table_result.tables.size(); + // catalog - { tuple->set_null(_tuple_desc->slots()[0]->null_indicator_offset()); } + { + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[0]); + } + } // schema { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); - str_slot->data = (char*)pool->allocate(db_name.size()); - str_slot->size = db_name.size(); - memcpy(const_cast(str_slot->data), db_name.c_str(), str_slot->size); + StringRef str = StringRef(db_name.c_str(), db_name.size()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[1]); + } } // name { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.name; - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memcpy failed."); + for (int i = 0; i < tables_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + const std::string* src = &tbl_status.name; + StringRef str = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str, _tuple_desc->slots()[2]); } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); } // definition { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* ddl_sql = &tbl_status.ddl_sql; - str_slot->size = ddl_sql->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memcpy failed."); + for (int i = 0; i < tables_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + const std::string* src = &tbl_status.ddl_sql; + StringRef str = StringRef(src->c_str(), src->length()); + fill_dest_column(block, &str, _tuple_desc->slots()[3]); } - memcpy(const_cast(str_slot->data), ddl_sql->c_str(), str_slot->size); } // check_option { - void* slot = tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - // This is from views in mysql const std::string check_option = "NONE"; - str_slot->size = check_option.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memcpy failed."); + StringRef str = StringRef(check_option.c_str(), check_option.length()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[4]); } - memcpy(const_cast(str_slot->data), check_option.c_str(), str_slot->size); } // is_updatable { - void* slot = tuple->get_slot(_tuple_desc->slots()[5]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); // This is from views in mysql const std::string is_updatable = "NO"; - str_slot->size = is_updatable.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memcpy failed."); + StringRef str = StringRef(is_updatable.c_str(), is_updatable.length()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[5]); } - memcpy(const_cast(str_slot->data), is_updatable.c_str(), str_slot->size); } // definer { - void* slot = tuple->get_slot(_tuple_desc->slots()[6]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); // This is from views in mysql const std::string definer = "root@%"; - str_slot->size = definer.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memcpy failed."); + StringRef str = StringRef(definer.c_str(), definer.length()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[6]); } - memcpy(const_cast(str_slot->data), definer.c_str(), str_slot->size); } // security_type { - void* slot = tuple->get_slot(_tuple_desc->slots()[7]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); // This is from views in mysql const std::string security_type = "DEFINER"; - str_slot->size = security_type.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); + StringRef str = StringRef(security_type.c_str(), security_type.length()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[7]); } - memcpy(const_cast(str_slot->data), security_type.c_str(), str_slot->size); } // character_set_client { - void* slot = tuple->get_slot(_tuple_desc->slots()[8]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); // This is from views in mysql const std::string encoding = "utf8"; - str_slot->size = encoding.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); + StringRef str = StringRef(encoding.c_str(), encoding.length()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _tuple_desc->slots()[8]); } - memcpy(const_cast(str_slot->data), encoding.c_str(), str_slot->size); } // collation_connection - { tuple->set_null(_tuple_desc->slots()[9]->null_indicator_offset()); } - _table_index++; - return Status::OK(); -} - -Status SchemaViewsScanner::get_new_table() { - TGetTablesParams table_params; - table_params.__set_db(_db_result.dbs[_db_index++]); - if (nullptr != _param->wild) { - table_params.__set_pattern(*(_param->wild)); - } - if (nullptr != _param->current_user_ident) { - table_params.__set_current_user_ident(*(_param->current_user_ident)); - } else { - if (nullptr != _param->user) { - table_params.__set_user(*(_param->user)); - } - if (nullptr != _param->user_ip) { - table_params.__set_user_ip(*(_param->user_ip)); + { + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, nullptr, _tuple_desc->slots()[9]); } } - table_params.__set_type("VIEW"); - - if (nullptr != _param->ip && 0 != _param->port) { - RETURN_IF_ERROR(SchemaHelper::list_table_status(*(_param->ip), _param->port, table_params, - &_table_result)); - } else { - return Status::InternalError("IP or port doesn't exists"); - } - _table_index = 0; return Status::OK(); } -Status SchemaViewsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("Used before initialized."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input pointer is nullptr."); - } - while (_table_index >= _table_result.tables.size()) { - if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); - } else { - *eos = true; - return Status::OK(); - } - } - *eos = false; - return fill_one_row(tuple, pool); -} - } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_views_scanner.h b/be/src/exec/schema_scanner/schema_views_scanner.h index 3222b67e897d59..1dc640b97562c1 100644 --- a/be/src/exec/schema_scanner/schema_views_scanner.h +++ b/be/src/exec/schema_scanner/schema_views_scanner.h @@ -28,14 +28,13 @@ class SchemaViewsScanner : public SchemaScanner { virtual ~SchemaViewsScanner(); virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _get_new_table(); + Status _fill_block_impl(vectorized::Block* block); int _db_index; - int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; static SchemaScanner::ColumnDesc _s_tbls_columns[]; diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index f9998f3fce3a21..704d6d09cf6cbb 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -260,113 +260,39 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, std::vector columns(_slot_num); bool schema_eos = false; - if (_schema_scanner->type() == TSchemaTableType::SCH_TABLES || - _schema_scanner->type() == TSchemaTableType::SCH_BACKENDS || - _schema_scanner->type() == TSchemaTableType::SCH_CHARSETS || - _schema_scanner->type() == TSchemaTableType::SCH_COLLATIONS || - _schema_scanner->type() == TSchemaTableType::SCH_COLUMNS) { - do { - block->clear(); - - for (int i = 0; i < _slot_num; ++i) { - int j = _index_map[i]; - const auto slot_desc = _src_tuple_desc->slots()[j]; - block->insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), - slot_desc->get_data_type_ptr(), - slot_desc->col_name())); - } - - while (true) { - RETURN_IF_CANCELLED(state); + do { + block->clear(); + + for (int i = 0; i < _slot_num; ++i) { + int j = _index_map[i]; + const auto slot_desc = _src_tuple_desc->slots()[j]; + block->insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), + slot_desc->get_data_type_ptr(), + slot_desc->col_name())); + } - // get all slots from schema table. - RETURN_IF_ERROR(_schema_scanner->get_next_block(block, &schema_eos)); + while (true) { + RETURN_IF_CANCELLED(state); - if (schema_eos) { - *eos = true; - break; - } + // get all slots from schema table. + RETURN_IF_ERROR(_schema_scanner->get_next_block(block, &schema_eos)); - if (block->rows() == state->batch_size()) { - break; - } + if (schema_eos) { + *eos = true; + break; } - if (block->rows()) { - RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, - _dest_tuple_desc->slots().size())); - VLOG_ROW << "VSchemaScanNode output rows: " << block->rows(); - } - } while (block->rows() == 0 && !(*eos)); - } else { - do { - bool mem_reuse = block->mem_reuse(); - DCHECK(block->rows() == 0); - - columns.resize(_slot_num); - for (int i = 0; i < _slot_num; ++i) { - if (mem_reuse) { - columns[i] = std::move(*block->get_by_position(i).column).mutate(); - } else { - columns[i] = _dest_tuple_desc->slots()[i]->get_empty_mutable_column(); - } - } - while (true) { - RETURN_IF_CANCELLED(state); - - // get all slots from schema table. - RETURN_IF_ERROR(_schema_scanner->get_next_row(_src_single_tuple, _tuple_pool.get(), - &schema_eos)); - if (schema_eos) { - *eos = true; - break; - } - // tuple project - project_tuple(); - - for (int i = 0; i < _slot_num; ++i) { - auto slot_desc = _dest_tuple_desc->slots()[i]; - if (!slot_desc->is_materialized()) { - continue; - } - - if (_dest_single_tuple->is_null(slot_desc->null_indicator_offset())) { - if (slot_desc->is_nullable()) { - auto* nullable_column = - reinterpret_cast(columns[i].get()); - nullable_column->insert_data(nullptr, 0); - } else { - return Status::InternalError( - "nonnull column contains NULL. table={}, column={}", - _table_name, slot_desc->col_name()); - } - } else { - RETURN_IF_ERROR(write_slot_to_vectorized_column( - _dest_single_tuple->get_slot(slot_desc->tuple_offset()), slot_desc, - &columns[i])); - } - } - if (columns[0]->size() == state->batch_size()) { - break; - } - } - if (!columns.empty() && !columns[0]->empty()) { - auto n_columns = 0; - if (!mem_reuse) { - for (const auto slot_desc : _dest_tuple_desc->slots()) { - block->insert(ColumnWithTypeAndName(std::move(columns[n_columns++]), - slot_desc->get_data_type_ptr(), - slot_desc->col_name())); - } - } else { - columns.clear(); - } - RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, - _dest_tuple_desc->slots().size())); - VLOG_ROW << "VSchemaScanNode output rows: " << block->rows(); + if (block->rows() == state->batch_size()) { + break; } - } while (block->rows() == 0 && !(*eos)); - } + } + + if (block->rows()) { + RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, + _dest_tuple_desc->slots().size())); + VLOG_ROW << "VSchemaScanNode output rows: " << block->rows(); + } + } while (block->rows() == 0 && !(*eos)); reached_limit(block, eos); return Status::OK(); From 3d5a7d2b1400cf17aaf138cd5ae9a330df6fac3e Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Wed, 11 Jan 2023 14:22:05 +0800 Subject: [PATCH 09/24] add regression-test --- .../query_p0/system/test_query_sys_tables.out | 90 +++++++++ .../system/test_query_sys_tables.groovy | 188 ++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 regression-test/data/query_p0/system/test_query_sys_tables.out create mode 100644 regression-test/suites/query_p0/system/test_query_sys_tables.groovy diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out new file mode 100644 index 00000000000000..f74207a8af78d6 --- /dev/null +++ b/regression-test/data/query_p0/system/test_query_sys_tables.out @@ -0,0 +1,90 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !backends -- +1 + +-- !charsets -- +utf8 utf8_general_ci UTF-8 Unicode 3 + +-- !collations -- +utf8_general_ci utf8 33 Yes Yes 1 + +-- !columns -- +internal test_query_sys_db_1 test_query_sys_tb_1 aaa 1 \N NO varchar 170 680 \N \N \N \N \N varchar(170) DUP 170 \N \N \N +internal test_query_sys_db_1 test_query_sys_tb_1 bbb 2 \N NO varchar 20 80 \N \N \N \N \N varchar(20) 20 \N \N \N +internal test_query_sys_db_1 test_query_sys_tb_1 ccc 3 \N YES int \N \N 10 0 \N \N \N int(11) 10 0 \N \N +internal test_query_sys_db_1 test_query_sys_tb_1 ddd 4 \N YES smallint \N \N 5 0 \N \N \N smallint(6) 5 0 \N \N +internal test_query_sys_db_2 test_query_sys_tb_2 aaa 1 \N NO varchar 170 680 \N \N \N \N \N varchar(170) DUP 170 \N \N \N +internal test_query_sys_db_2 test_query_sys_tb_2 bbb 2 \N NO varchar 20 80 \N \N \N \N \N varchar(20) 20 \N \N \N +internal test_query_sys_db_2 test_query_sys_tb_2 ccc 3 \N YES int \N \N 10 0 \N \N \N int(11) 10 0 \N \N +internal test_query_sys_db_2 test_query_sys_tb_2 ddd 4 \N YES smallint \N \N 5 0 \N \N \N smallint(6) 5 0 \N \N +internal test_query_sys_db_3 test_query_sys_tb_3 aaa 1 \N NO varchar 170 680 \N \N \N \N \N varchar(170) DUP 170 \N \N \N +internal test_query_sys_db_3 test_query_sys_tb_3 bbb 2 \N NO varchar 20 80 \N \N \N \N \N varchar(20) 20 \N \N \N +internal test_query_sys_db_3 test_query_sys_tb_3 ccc 3 \N YES int \N \N 10 0 \N \N \N int(11) 10 0 \N \N +internal test_query_sys_db_3 test_query_sys_tb_3 ddd 4 \N YES smallint \N \N 5 0 \N \N \N smallint(6) 5 0 \N \N + +-- !files -- + +-- !partitions -- + +-- !rowsets1 -- +19 + +-- !rowsets2 -- +0200000000000002d1408c85500eeb5227fb0d497599f39d 11018 0 1673417231 +020000000000000ebd4f7558454abaf995adf4bf4ca9f5af 10325 0 1671976020 +0200000000000001bd4f7558454abaf995adf4bf4ca9f5af 10296 0 1671976020 +0200000000000003d1408c85500eeb5227fb0d497599f39d 11023 0 1673417231 +0200000000000002bd4f7558454abaf995adf4bf4ca9f5af 10298 0 1671976020 +0200000000000003bd4f7558454abaf995adf4bf4ca9f5af 10300 0 1671976020 +0200000000000008bd4f7558454abaf995adf4bf4ca9f5af 10313 0 1671976020 +0200000000000010bd4f7558454abaf995adf4bf4ca9f5af 10331 1442 1671976082 +020000000000000fbd4f7558454abaf995adf4bf4ca9f5af 10331 0 1671976062 +0200000000000004bd4f7558454abaf995adf4bf4ca9f5af 10302 0 1671976020 +0200000000000009bd4f7558454abaf995adf4bf4ca9f5af 10315 0 1671976020 +0200000000000005bd4f7558454abaf995adf4bf4ca9f5af 10304 0 1671976020 +0200000000000001d1408c85500eeb5227fb0d497599f39d 11013 0 1673417231 +020000000000000abd4f7558454abaf995adf4bf4ca9f5af 10317 0 1671976020 +0200000000000006bd4f7558454abaf995adf4bf4ca9f5af 10306 0 1671976020 +020000000000000bbd4f7558454abaf995adf4bf4ca9f5af 10319 0 1671976020 +0200000000000007bd4f7558454abaf995adf4bf4ca9f5af 10308 0 1671976020 +020000000000000cbd4f7558454abaf995adf4bf4ca9f5af 10321 0 1671976020 +020000000000000dbd4f7558454abaf995adf4bf4ca9f5af 10323 0 1671976020 + +-- !schema_privileges -- + +-- !schemata -- +internal utf8 \N +internal utf8 \N +internal utf8 \N + +-- !statistics -- + +-- !table_privileges -- + +-- !tables -- +internal test_query_sys_tb_1 BASE TABLE 0 \N \N 2023-01-11T14:07:11 +internal test_query_sys_tb_2 BASE TABLE 0 \N \N 2023-01-11T14:07:11 +internal test_query_sys_tb_3 BASE TABLE 0 \N \N 2023-01-11T14:07:11 + +-- !user_privileges -- +'root'@'%' def SELECT YES +'root'@'%' def INSERT YES +'root'@'%' def ALTER YES +'root'@'%' def CREATE YES +'root'@'%' def DROP YES +'root'@'%' def USAGE YES +'admin'@'%' def SELECT YES +'admin'@'%' def INSERT YES +'admin'@'%' def ALTER YES +'admin'@'%' def CREATE YES +'admin'@'%' def DROP YES +'admin'@'%' def USAGE YES + +-- !session_variables -- +124 + +-- !session_variables -- +124 + +-- !views -- + diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy new file mode 100644 index 00000000000000..695084f0e1bab6 --- /dev/null +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_query_sys_tables", "query,p0") { + def dbName1 = "test_query_sys_db_1" + def dbName2 = "test_query_sys_db_2" + def dbName3 = "test_query_sys_db_3" + def tbName1 = "test_query_sys_tb_1" + def tbName2 = "test_query_sys_tb_2" + def tbName3 = "test_query_sys_tb_3" + + // test backends + sql("use information_schema") + qt_backends("select count(*) from backends") + + // test charsets + sql("use information_schema") + qt_charsets("select * from character_sets") + + // test collations + sql("use information_schema") + qt_collations("select * from collations") + + // test columns + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql("use ${dbName2}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql("use ${dbName3}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql("use information_schema") + qt_columns("select * from columns where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'") + + // test files + sql("use information_schema") + qt_files("select * from files") + + // test partitions + sql("use information_schema") + qt_partitions("select * from partitions") + + // test rowsets + sql("use information_schema") + qt_rowsets1("select count(*) from rowsets"); + qt_rowsets2("select ROWSET_ID, TABLET_ID, DATA_DISK_SIZE, CREATION_TIME from rowsets") + + // test schema privileges + sql("use information_schema") + qt_schema_privileges("select * from schema_privileges") + + // test schemata + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + + sql("use information_schema") + qt_schemata("select CATALOG_NAME, DEFAULT_CHARACTER_SET_NAME, SQL_PATH from schemata where SCHEMA_NAME = '${dbName1}' or SCHEMA_NAME = '${dbName2}' or SCHEMA_NAME = '${dbName3}'"); + + // test statistics + sql("use information_schema") + qt_statistics("select * from statistics") + + // test table privileges + sql("use information_schema") + qt_table_privileges("select * from table_privileges") + + // test tables + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql("use ${dbName2}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql("use ${dbName3}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql("use information_schema") + qt_tables("select TABLE_CATALOG, TABLE_NAME, TABLE_TYPE, AVG_ROW_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, CREATE_TIME from tables where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'"); + + // test user privileges + sql("use information_schema") + qt_user_privileges("select * from user_privileges") + + // test variables + // session_variables + sql("use information_schema") + qt_session_variables("select count(*) from session_variables") + // global_variables + sql("use information_schema") + qt_session_variables("select count(*) from global_variables") + + // test views + sql("use information_schema") + qt_views("select * from views") +} From 8249438dc073e56fbaa745fefe8c3b7a22ed545d Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Thu, 12 Jan 2023 17:11:51 +0800 Subject: [PATCH 10/24] fix rowsets_scanner --- .../schema_scanner/schema_rowsets_scanner.cpp | 39 +++++++++++-------- .../schema_scanner/schema_rowsets_scanner.h | 2 + be/src/vec/exec/vschema_scan_node.cpp | 22 +++++------ 3 files changed, 34 insertions(+), 29 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp index 599b8e0d8fffa6..1110a7fe47d3a6 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp @@ -53,7 +53,8 @@ SchemaRowsetsScanner::SchemaRowsetsScanner() : SchemaScanner(_s_tbls_columns, sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), TSchemaTableType::SCH_ROWSETS), - backend_id_(0) {}; + backend_id_(0), + _rowsets_idx(0) {}; Status SchemaRowsetsScanner::start(RuntimeState* state) { if (!_is_init) { @@ -90,25 +91,28 @@ Status SchemaRowsetsScanner::get_next_block(vectorized::Block* block, bool* eos) return Status::InternalError("input pointer is nullptr."); } - *eos = true; - if (!rowsets_.size()) { + if (_rowsets_idx >= rowsets_.size()) { + *eos = true; return Status::OK(); } + *eos = false; return _fill_block_impl(block); } Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { - auto rowsets_num = rowsets_.size(); + size_t fill_rowsets_num = std::min(1000ul, rowsets_.size() - _rowsets_idx); + auto fill_idx_begin = _rowsets_idx; + auto fill_idx_end = _rowsets_idx + fill_rowsets_num; // BACKEND_ID { int64_t src = backend_id_; - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { fill_dest_column(block, &src, _tuple_desc->slots()[0]); } } // ROWSET_ID { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; std::string rowset_id = rowset->rowset_id().to_string(); StringRef str = StringRef(rowset_id.c_str(), rowset_id.size()); @@ -117,7 +121,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // TABLET_ID { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->rowset_meta()->tablet_id(); fill_dest_column(block, &src, _tuple_desc->slots()[2]); @@ -125,7 +129,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // ROWSET_NUM_ROWS { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->num_rows(); fill_dest_column(block, &src, _tuple_desc->slots()[3]); @@ -133,7 +137,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // TXN_ID { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->txn_id(); fill_dest_column(block, &src, _tuple_desc->slots()[4]); @@ -141,7 +145,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // NUM_SEGMENTS { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->num_segments(); fill_dest_column(block, &src, _tuple_desc->slots()[5]); @@ -149,7 +153,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // START_VERSION { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->start_version(); fill_dest_column(block, &src, _tuple_desc->slots()[6]); @@ -157,7 +161,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // END_VERSION { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->end_version(); fill_dest_column(block, &src, _tuple_desc->slots()[7]); @@ -165,7 +169,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // INDEX_DISK_SIZE { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; size_t src = rowset->index_disk_size(); fill_dest_column(block, &src, _tuple_desc->slots()[8]); @@ -173,7 +177,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // DATA_DISK_SIZE { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; size_t src = rowset->data_disk_size(); fill_dest_column(block, &src, _tuple_desc->slots()[9]); @@ -181,7 +185,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // CREATION_TIME { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; size_t src = rowset->creation_time(); fill_dest_column(block, &src, _tuple_desc->slots()[10]); @@ -189,7 +193,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // OLDEST_WRITE_TIMESTAMP { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; size_t src = rowset->oldest_write_timestamp(); fill_dest_column(block, &src, _tuple_desc->slots()[11]); @@ -197,12 +201,13 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { } // NEWEST_WRITE_TIMESTAMP { - for (int i = 0; i < rowsets_num; ++i) { + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; size_t src = rowset->newest_write_timestamp(); fill_dest_column(block, &src, _tuple_desc->slots()[12]); } } + _rowsets_idx += fill_rowsets_num; return Status::OK(); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.h b/be/src/exec/schema_scanner/schema_rowsets_scanner.h index 92341b7197be41..273abe3a950332 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include @@ -41,6 +42,7 @@ class SchemaRowsetsScanner : public SchemaScanner { static SchemaScanner::ColumnDesc _s_tbls_columns[]; int64_t backend_id_ = 0; + size_t _rowsets_idx = 0; std::vector rowsets_; }; } // namespace doris diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index 704d6d09cf6cbb..ff2849160c0e4b 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -257,20 +257,18 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, return Status::InternalError("used before initialize."); } RETURN_IF_CANCELLED(state); - std::vector columns(_slot_num); bool schema_eos = false; - do { - block->clear(); - - for (int i = 0; i < _slot_num; ++i) { - int j = _index_map[i]; - const auto slot_desc = _src_tuple_desc->slots()[j]; - block->insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), - slot_desc->get_data_type_ptr(), - slot_desc->col_name())); - } + block->clear(); + for (int i = 0; i < _slot_num; ++i) { + int j = _index_map[i]; + const auto slot_desc = _src_tuple_desc->slots()[j]; + block->insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), + slot_desc->get_data_type_ptr(), slot_desc->col_name())); + } + + do { while (true) { RETURN_IF_CANCELLED(state); @@ -282,7 +280,7 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, break; } - if (block->rows() == state->batch_size()) { + if (block->rows() >= state->batch_size()) { break; } } From 945b00830a2b394aeaeadd45f17c652f99438c7e Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Thu, 12 Jan 2023 19:53:08 +0800 Subject: [PATCH 11/24] fix --- be/src/exec/schema_scanner/schema_charsets_scanner.h | 2 +- be/src/exec/schema_scanner/schema_collations_scanner.h | 2 +- be/src/exec/schema_scanner/schema_columns_scanner.h | 4 ++-- be/src/exec/schema_scanner/schema_dummy_scanner.h | 4 ++-- be/src/exec/schema_scanner/schema_files_scanner.h | 4 ++-- be/src/exec/schema_scanner/schema_partitions_scanner.h | 4 ++-- be/src/exec/schema_scanner/schema_schema_privileges_scanner.h | 4 ++-- be/src/exec/schema_scanner/schema_schemata_scanner.h | 4 ++-- be/src/exec/schema_scanner/schema_statistics_scanner.h | 2 +- be/src/exec/schema_scanner/schema_table_privileges_scanner.h | 4 ++-- be/src/exec/schema_scanner/schema_tables_scanner.h | 4 ++-- be/src/exec/schema_scanner/schema_user_privileges_scanner.h | 4 ++-- be/src/exec/schema_scanner/schema_variables_scanner.h | 4 ++-- be/src/exec/schema_scanner/schema_views_scanner.h | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.h b/be/src/exec/schema_scanner/schema_charsets_scanner.h index 765262cfecc6c3..eb5f61bfc63421 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.h @@ -26,7 +26,7 @@ namespace doris { class SchemaCharsetsScanner : public SchemaScanner { public: SchemaCharsetsScanner(); - virtual ~SchemaCharsetsScanner(); + ~SchemaCharsetsScanner() override; Status get_next_block(vectorized::Block* block, bool* eos) override; diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.h b/be/src/exec/schema_scanner/schema_collations_scanner.h index ce81446aca0371..58f0ee54de4901 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.h +++ b/be/src/exec/schema_scanner/schema_collations_scanner.h @@ -26,7 +26,7 @@ namespace doris { class SchemaCollationsScanner : public SchemaScanner { public: SchemaCollationsScanner(); - virtual ~SchemaCollationsScanner(); + ~SchemaCollationsScanner() override; Status get_next_block(vectorized::Block* block, bool* eos) override; diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.h b/be/src/exec/schema_scanner/schema_columns_scanner.h index 2a14db79a4357c..6f50bfd58510d7 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.h +++ b/be/src/exec/schema_scanner/schema_columns_scanner.h @@ -27,8 +27,8 @@ namespace doris { class SchemaColumnsScanner : public SchemaScanner { public: SchemaColumnsScanner(); - virtual ~SchemaColumnsScanner(); - virtual Status start(RuntimeState* state); + ~SchemaColumnsScanner() override; + Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; private: diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.h b/be/src/exec/schema_scanner/schema_dummy_scanner.h index 1bef2e547f1488..d48466e83a0827 100644 --- a/be/src/exec/schema_scanner/schema_dummy_scanner.h +++ b/be/src/exec/schema_scanner/schema_dummy_scanner.h @@ -24,8 +24,8 @@ namespace doris { class SchemaDummyScanner : public SchemaScanner { public: SchemaDummyScanner(); - virtual ~SchemaDummyScanner(); - virtual Status start(RuntimeState* state = nullptr); + ~SchemaDummyScanner() override; + Status start(RuntimeState* state = nullptr) override; Status get_next_block(vectorized::Block* block, bool* eos) override; }; diff --git a/be/src/exec/schema_scanner/schema_files_scanner.h b/be/src/exec/schema_scanner/schema_files_scanner.h index 831a748cf70528..834b98908ae13c 100644 --- a/be/src/exec/schema_scanner/schema_files_scanner.h +++ b/be/src/exec/schema_scanner/schema_files_scanner.h @@ -25,9 +25,9 @@ namespace doris { class SchemaFilesScanner : public SchemaScanner { public: SchemaFilesScanner(); - virtual ~SchemaFilesScanner(); + ~SchemaFilesScanner() override; - virtual Status start(RuntimeState* state); + Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; int _db_index; diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.h b/be/src/exec/schema_scanner/schema_partitions_scanner.h index fc8490afb173cb..6fe1baf480deb1 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.h +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.h @@ -25,9 +25,9 @@ namespace doris { class SchemaPartitionsScanner : public SchemaScanner { public: SchemaPartitionsScanner(); - virtual ~SchemaPartitionsScanner(); + ~SchemaPartitionsScanner() override; - virtual Status start(RuntimeState* state); + Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; int _db_index; diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h index 0782983f76b023..3222e9d81ac129 100644 --- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h @@ -25,9 +25,9 @@ namespace doris { class SchemaSchemaPrivilegesScanner : public SchemaScanner { public: SchemaSchemaPrivilegesScanner(); - virtual ~SchemaSchemaPrivilegesScanner(); + ~SchemaSchemaPrivilegesScanner() override; - virtual Status start(RuntimeState* state); + Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; private: diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.h b/be/src/exec/schema_scanner/schema_schemata_scanner.h index 26d97b14c7ceef..0077520441dd02 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.h +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.h @@ -25,9 +25,9 @@ namespace doris { class SchemaSchemataScanner : public SchemaScanner { public: SchemaSchemataScanner(); - virtual ~SchemaSchemataScanner(); + ~SchemaSchemataScanner() override; - virtual Status start(RuntimeState* state); + Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; private: diff --git a/be/src/exec/schema_scanner/schema_statistics_scanner.h b/be/src/exec/schema_scanner/schema_statistics_scanner.h index 7953a57987132c..ebbcc27216fead 100644 --- a/be/src/exec/schema_scanner/schema_statistics_scanner.h +++ b/be/src/exec/schema_scanner/schema_statistics_scanner.h @@ -23,7 +23,7 @@ namespace doris { class SchemaStatisticsScanner : public SchemaScanner { public: SchemaStatisticsScanner(); - virtual ~SchemaStatisticsScanner(); + ~SchemaStatisticsScanner() override; private: static SchemaScanner::ColumnDesc _s_cols_statistics[]; diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h index cd8f40257cfa1c..82629522b8c2fc 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h @@ -25,9 +25,9 @@ namespace doris { class SchemaTablePrivilegesScanner : public SchemaScanner { public: SchemaTablePrivilegesScanner(); - virtual ~SchemaTablePrivilegesScanner(); + ~SchemaTablePrivilegesScanner() override; - virtual Status start(RuntimeState* state); + Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; private: diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h index 6b7f2a65098cd0..933d1f8e9a11a3 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_tables_scanner.h @@ -27,9 +27,9 @@ namespace doris { class SchemaTablesScanner : public SchemaScanner { public: SchemaTablesScanner(); - virtual ~SchemaTablesScanner(); + ~SchemaTablesScanner() override; - virtual Status start(RuntimeState* state); + Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; private: diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h index 4ec35e4cfdf417..a4b5313be51f78 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h @@ -25,9 +25,9 @@ namespace doris { class SchemaUserPrivilegesScanner : public SchemaScanner { public: SchemaUserPrivilegesScanner(); - virtual ~SchemaUserPrivilegesScanner(); + ~SchemaUserPrivilegesScanner() override; - virtual Status start(RuntimeState* state); + Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; private: diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.h b/be/src/exec/schema_scanner/schema_variables_scanner.h index 21111ae9fe7087..257858684063f4 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.h +++ b/be/src/exec/schema_scanner/schema_variables_scanner.h @@ -28,9 +28,9 @@ namespace doris { class SchemaVariablesScanner : public SchemaScanner { public: SchemaVariablesScanner(TVarType::type type); - virtual ~SchemaVariablesScanner(); + ~SchemaVariablesScanner() override; - virtual Status start(RuntimeState* state); + Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; private: diff --git a/be/src/exec/schema_scanner/schema_views_scanner.h b/be/src/exec/schema_scanner/schema_views_scanner.h index 1dc640b97562c1..968f99ca299fa4 100644 --- a/be/src/exec/schema_scanner/schema_views_scanner.h +++ b/be/src/exec/schema_scanner/schema_views_scanner.h @@ -25,9 +25,9 @@ namespace doris { class SchemaViewsScanner : public SchemaScanner { public: SchemaViewsScanner(); - virtual ~SchemaViewsScanner(); + ~SchemaViewsScanner() override; - virtual Status start(RuntimeState* state); + Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; private: From 941e4105bf77e55c7cbe2d095a3f7d14d833915e Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Mon, 16 Jan 2023 15:34:00 +0800 Subject: [PATCH 12/24] fix again --- be/src/exec/schema_scanner.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index f8ce4b1978ffe8..f087eaaad41238 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -33,6 +33,7 @@ #include "exec/schema_scanner/schema_variables_scanner.h" #include "exec/schema_scanner/schema_views_scanner.h" #include "runtime/define_primitive_type.h" +#include "vec/columns/column.h" #include "vec/common/string_ref.h" #include "vec/core/block.h" @@ -237,8 +238,9 @@ Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, if (!block->has(slot_desc->col_name())) { return Status::OK(); } - vectorized::IColumn* col_ptr = const_cast( - block->get_by_name(slot_desc->col_name()).column.get()); + vectorized::MutableColumnPtr column_ptr = + std::move(*block->get_by_name(slot_desc->col_name()).column).mutate(); + vectorized::IColumn* col_ptr = column_ptr.get(); if (data == nullptr) { if (!slot_desc->is_nullable()) { @@ -263,9 +265,9 @@ Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, case TYPE_VARCHAR: case TYPE_CHAR: case TYPE_STRING: { - StringValue* str_slot = reinterpret_cast(data); - reinterpret_cast(col_ptr)->insert_data(str_slot->ptr, - str_slot->len); + StringRef* str_slot = reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data(str_slot->data, + str_slot->size); break; } From 296b24d95f70a41a0c520ff16f48224baa7c004f Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Mon, 16 Jan 2023 20:20:28 +0800 Subject: [PATCH 13/24] assume_mutable replace mutable --- be/src/exec/schema_scanner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index f087eaaad41238..86242f43673ece 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -239,7 +239,7 @@ Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, return Status::OK(); } vectorized::MutableColumnPtr column_ptr = - std::move(*block->get_by_name(slot_desc->col_name()).column).mutate(); + std::move(*block->get_by_name(slot_desc->col_name()).column).assume_mutable(); vectorized::IColumn* col_ptr = column_ptr.get(); if (data == nullptr) { From f0a04e3ef6c4e2dac470d64a96f1e07ea2353999 Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Tue, 17 Jan 2023 19:31:28 +0800 Subject: [PATCH 14/24] fix regression test error and add sys tables test --- be/src/exec/schema_scanner.cpp | 10 ++- be/src/vec/exec/vschema_scan_node.cpp | 7 +- .../query_p0/system/test_query_sys_tables.out | Bin 3771 -> 2510 bytes .../system/test_query_sys_tables.groovy | 66 ++++++++---------- 4 files changed, 37 insertions(+), 46 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 86242f43673ece..37eccaeefe49a2 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -235,7 +235,7 @@ Status SchemaScanner::create_tuple_desc(ObjectPool* pool) { Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, const SlotDescriptor* slot_desc) { - if (!block->has(slot_desc->col_name())) { + if (!block->has(slot_desc->col_name()) || !slot_desc->is_materialized()) { return Status::OK(); } vectorized::MutableColumnPtr column_ptr = @@ -251,11 +251,9 @@ Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, nullable_column->insert_data(nullptr, 0); return Status::OK(); } - if (slot_desc->is_nullable()) { - auto* nullable_column = reinterpret_cast(col_ptr); - nullable_column->get_null_map_data().push_back(0); - col_ptr = &nullable_column->get_nested_column(); - } + auto* nullable_column = reinterpret_cast(col_ptr); + nullable_column->get_null_map_data().push_back(0); + col_ptr = &nullable_column->get_nested_column(); switch (slot_desc->type().type) { case TYPE_HLL: { HyperLogLog* hll_slot = reinterpret_cast(data); diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index ff2849160c0e4b..cf50896ca6828b 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -262,10 +262,9 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, block->clear(); for (int i = 0; i < _slot_num; ++i) { - int j = _index_map[i]; - const auto slot_desc = _src_tuple_desc->slots()[j]; - block->insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), - slot_desc->get_data_type_ptr(), slot_desc->col_name())); + auto dest_slot_desc = _dest_tuple_desc->slots()[i]; + block->insert(ColumnWithTypeAndName(dest_slot_desc->get_empty_mutable_column(), + dest_slot_desc->get_data_type_ptr(), dest_slot_desc->col_name())); } do { diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out index f74207a8af78d6045a9945e926f90c9ca52ecbaf..14fcdeb5d28b013380fd12724c24ece46436edec 100644 GIT binary patch delta 360 zcmdljdro*mBqK8eC&OetX06G;7{w;fVl-t1@_`~38FQd4d!}M8uox$&DW}2Y23DoX z=a|JNe`KgV%Ur(AL2GvW#EkQgMU^8U%I}$dz0Gx9(c! literal 3771 zcmcgvU2mH(6n%Dn1$&sTQ*Iq+Nwz&DpiUMHVrC)4V*^#_cJD6q6E3y zQk2ho@Hxl6_O-EXYcfwOD@ik9sc*KF#e&H+W?8njo<%N7CI$a$Jr+x=nitEATQ7OB zvQ~2@t+fjBlv`<{f38?AEm>HCr|MhL4s6@{6tVb4Ong*- z%o??97N=-B`EL8xG*9D#3q&`qiY!ysihS4Zu!-iO6g`Sczqe9Fwp=tV(p-u%XBm>B zlC$4SQLbmzx|;E52JM)vV~jBb=;;>S-l11k)~*qF7!7?~M-N{c_*}tX(Dn2lsjI$q zd(YtA;!zYK(h?^a9hW9bxxUm*f3F(HF`{+*$8e04=M5ELv}dj3=Kzh`9(jKkIp@gP zid73W<@Sp1q2LxfA2y;puuqe8 z^+Y-C7W)r5)n&ilWOVJ;|9-Jjrn_>kLuIk5w*}B!MbK3JO^nnrOz`1FI5^mkeb>cU zh{z?xOCrpjz;lBjp#et#7<^SpVCYc?JVK$(AtLT1p6mLK>#&F=LEti;I8oxnEJ$3I zAi$K=1xIl};0=|BHU}7RkYL!$K~4HXXm4MLk&n%I9O(-+;GnqP4vvq9Z^8kazI6Kl zY{EeSf#Z;!6`!(mN!92?z8~GuMOzJH0-@WSVVLb7aXd6CdF9 z`WTvU;Ql2OnQ%bR>%%kQ!2L@mHsK)9>w_C{w3dvnnN>U&3pV>zrmw2JJ|7g=29d0; zoV(+vJ|;ii8Uz}4sZQLLdLG!^BLkILF4cyyqeU<=0OYj zc{>gtI+4-MBz)* Date: Tue, 17 Jan 2023 19:33:56 +0800 Subject: [PATCH 15/24] fix code format --- be/src/vec/exec/vschema_scan_node.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index cf50896ca6828b..28ce55787c9b99 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -264,7 +264,8 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, for (int i = 0; i < _slot_num; ++i) { auto dest_slot_desc = _dest_tuple_desc->slots()[i]; block->insert(ColumnWithTypeAndName(dest_slot_desc->get_empty_mutable_column(), - dest_slot_desc->get_data_type_ptr(), dest_slot_desc->col_name())); + dest_slot_desc->get_data_type_ptr(), + dest_slot_desc->col_name())); } do { From 849f04e9787700976672bb8918c86669a321b586 Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Tue, 17 Jan 2023 19:46:57 +0800 Subject: [PATCH 16/24] add License --- .../system/test_query_sys_tables.groovy | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy index 646995e10006d8..ff034f46fd7131 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + suite("test_query_sys_tables", "query,p0") { def dbName1 = "test_query_sys_db_1" def dbName2 = "test_query_sys_db_2" From 8036bbb807fbe196ed9833b81bdd7234c2534302 Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Wed, 18 Jan 2023 13:53:41 +0800 Subject: [PATCH 17/24] fix regression test --- .../query_p0/system/test_query_sys_tables.out | Bin 2510 -> 1555 bytes .../system/test_query_sys_tables.groovy | 46 ++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out index 14fcdeb5d28b013380fd12724c24ece46436edec..645d726d6be8f861a5bdff662ef26723e813b650 100644 GIT binary patch literal 1555 zcmd5+U2mH(6n!?o!h5KaJ}?Ect$QqOy(v;er;v53Gzl5(G*`S8j`(2wRVU#Sd zMZ(wIb9{YlAME>dxmJd*R3Rxw-auV)P&qG(9et9O)EuPvNFVElnswb2f<9OEmTuP^ z=nj)Y3944EyX6&7sHqeRBbN1ipMJ=A{wynDsPB8A8|k@hzUJCU=x6Jqz}f5SZN4en zq^dybiWh`)P6D#w+LqeKad0)cB-nTdUY2Df+~fN=4v`jiSb3h4DZxSZ9R~px>7YUg z@`;!dcg0fno$;4T^7%NE)pM%CJ5>8#_Rsmj8qgiO`+DIT)WHWXW%9KwdHSN&2B}XS za*F`B2E*-d-2u`-`lpe4mztePMV1ESclVHF$RTkKaJ*B} zz)HJ`5=NKN&76@5&hu8vM#(LCn6Y0c@Z(akUU(RxC z2)QB&o3q;`d1Q&#LuPTp7K2DMU$Vu}klO`|mTU;QTipK`%KuEF@2>;6C{;BqtOs?K z^`MTl9@5p;gO1{Qz)@Xm9BB+*ZYf?`${NP`V#^hzph{UcK&BIXUR`#c9305j9`@i+nkUREm7UO9&u=?s8{ulfF*&F}> literal 2510 zcmcguU2obj6z%KfS9lK<>>(3LwvNZrl1`Nntsp!gO^Te@6st+x#SVynU)zBs%DfMz z0be2aoa=L=W8G`VLE(l8v}P$q>WWLrGb~wx({ztsXif_(Dfxs}yhLKd%aowkobS+X zgC*LlAYlZtwfVclxkQp9N|?H;WqXc;KEybAr8yDEae8jggpv&|1eGRK%Jm5JPgJR; zQNmyuJ~$(^%vr(-1^hjBq>f>X!G-TLm_5N37rJV2zYgJMG}O`d=Kv2B{1GP0XSh;tLq4*4C(Fli z3|>dx8^W)}J*BO6m9pafyKs^u;OqQVumDx|VJUYA&8j*+aNSz40Z)HWLI`|5>WNHk zy+-s)Tl0twZfe~c|DCL7vDdqty>|Hv*lQO)pS^ayv)k)i?Da2auU-BE_S%KdXRlrF z|JZBhf}TRAxtqH;Tu4^;dlM4LhGsZ=E7(?1Us~h}kyuZey3^{JaHSf{UqvbbsV0n2 z^RFpG;y9%?Ndp*OE!deXW{TX67Xb>#chlh5%9?V1=6i4&-?#D2qm+UcowEA{=dyp> zzwQH}Ygh!+;6Bt{zSlt>XNzENMaI)GnA=kB=fOA(Y{+E(^lXbS7vpb-1WYoP+Y9SJ z?aDe(tF#W%uB`(##dUzDx@u!kA=J?$QiGo94PkuWVJ0KVGRjK{{!qQDlZyB=*{#W2 pMt5d4HE=uop*fjo5~y>?EYOD$ZR71VOoE5UnR3G%ebg@Ry#qm>;_(0g diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy index ff034f46fd7131..3292e7f83008c2 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -25,15 +25,15 @@ suite("test_query_sys_tables", "query,p0") { // test backends sql("use information_schema") - qt_backends("select count(*) from backends") + qt_backends("select count(*) >= 1 from backends") // test charsets sql("use information_schema") - qt_charsets("select * from character_sets") + qt_charsets("select count(*) >= 1 from character_sets") // test collations sql("use information_schema") - qt_collations("select * from collations") + qt_collations("select count(*) >= 1 from collations") // test columns // create test dbs @@ -51,7 +51,8 @@ suite("test_query_sys_tables", "query,p0") { ) DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_num" = "1", + "disable_auto_compaction" = "true" ); """ sql("use ${dbName2}") @@ -64,7 +65,8 @@ suite("test_query_sys_tables", "query,p0") { ) DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_num" = "1", + "disable_auto_compaction" = "true" ); """ sql("use ${dbName3}") @@ -77,22 +79,35 @@ suite("test_query_sys_tables", "query,p0") { ) DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_num" = "1", + "disable_auto_compaction" = "true" ); """ sql("use information_schema") - qt_columns("select * from columns where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'") + qt_columns("select TABLE_CATALOG, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, COLUMN_TYPE, COLUMN_SIZE from columns where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'") // test files + sql """ + CREATE FILE "test_file" + IN ${dbName1} + PROPERTIES + ( + "url" = "http://bit-images.bj.bcebos.com/bit2/index.html", + "catalog" = "test" + ); + """ sql("use information_schema") - qt_files("select * from files") + qt_files("select FILE_ID, FILE_NAME, FILE_TYPE from files where TABLE_SCHEMA = '${dbName1}'") + sql """ + DROP FILE "test_file" from test_query_sys_db_1 properties("catalog" = "test") + """ // test partitions sql("use information_schema") qt_partitions("select * from partitions") // test rowsets - // have no tablet tables, add this later + // have no tablet system table, add this later // sql("use information_schema") // qt_rowsets1("select count(*) from rowsets"); // qt_rowsets2("select ROWSET_ID, TABLET_ID, DATA_DISK_SIZE, CREATION_TIME from rowsets") @@ -109,7 +124,7 @@ suite("test_query_sys_tables", "query,p0") { sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") sql("use information_schema") - qt_schemata("select CATALOG_NAME, DEFAULT_CHARACTER_SET_NAME, SQL_PATH from schemata where SCHEMA_NAME = '${dbName1}' or SCHEMA_NAME = '${dbName2}' or SCHEMA_NAME = '${dbName3}'"); + qt_schemata("select CATALOG_NAME, SCHEMA_NAME, SQL_PATH from schemata where SCHEMA_NAME = '${dbName1}' or SCHEMA_NAME = '${dbName2}' or SCHEMA_NAME = '${dbName3}'"); // test statistics sql("use information_schema") @@ -140,7 +155,8 @@ suite("test_query_sys_tables", "query,p0") { ) DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_num" = "1", + "disable_auto_compaction" = "true" ); """ sql("use ${dbName2}") @@ -153,7 +169,8 @@ suite("test_query_sys_tables", "query,p0") { ) DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_num" = "1", + "disable_auto_compaction" = "true" ); """ sql("use ${dbName3}") @@ -166,7 +183,8 @@ suite("test_query_sys_tables", "query,p0") { ) DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" + "replication_num" = "1", + "disable_auto_compaction" = "true" ); """ @@ -190,7 +208,7 @@ suite("test_query_sys_tables", "query,p0") { // test views sql("use ${dbName1}") sql """ - CREATE VIEW ${dbName1}.test_view (a) + CREATE VIEW IF NOT EXISTS ${dbName1}.test_view (a) AS SELECT ccc as a FROM ${tbName1} """ From 52e59cf12a2c14a3912bcdf6b782d4c7a3781305 Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Thu, 19 Jan 2023 01:14:28 +0800 Subject: [PATCH 18/24] fix regression again --- .../query_p0/system/test_query_sys_tables.out | 33 ++---------- .../system/test_query_sys_tables.groovy | 52 ++++++------------- 2 files changed, 20 insertions(+), 65 deletions(-) diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out index 645d726d6be8f8..6972d35ff6fcd3 100644 --- a/regression-test/data/query_p0/system/test_query_sys_tables.out +++ b/regression-test/data/query_p0/system/test_query_sys_tables.out @@ -13,7 +13,6 @@ internal aaa 1 varchar varchar(170) 170 internal bbb 2 varchar varchar(20) 20 internal ccc 3 int int(11) 10 internal ddd 4 smallint smallint(6) 5 -internal a 1 int int(11) 10 internal aaa 1 varchar varchar(170) 170 internal bbb 2 varchar varchar(20) 20 internal ccc 3 int int(11) 10 @@ -23,47 +22,25 @@ internal bbb 2 varchar varchar(20) 20 internal ccc 3 int int(11) 10 internal ddd 4 smallint smallint(6) 5 --- !files -- - --- !partitions -- - --- !schema_privileges -- - -- !schemata -- internal test_query_sys_db_1 \N internal test_query_sys_db_2 \N internal test_query_sys_db_3 \N --- !statistics -- - --- !table_privileges -- - -- !tables -- internal test_query_sys_tb_1 BASE TABLE 0 \N \N -internal test_view VIEW 0 \N \N internal test_query_sys_tb_2 BASE TABLE 0 \N \N internal test_query_sys_tb_3 BASE TABLE 0 \N \N --- !user_privileges -- -'root'@'%' def SELECT YES -'root'@'%' def INSERT YES -'root'@'%' def ALTER YES -'root'@'%' def CREATE YES -'root'@'%' def DROP YES -'root'@'%' def USAGE YES -'admin'@'%' def SELECT YES -'admin'@'%' def INSERT YES -'admin'@'%' def ALTER YES -'admin'@'%' def CREATE YES -'admin'@'%' def DROP YES -'admin'@'%' def USAGE YES - -- !session_variables -- wait_timeout 30000 --- !session_variables -- +-- !global_variables -- wait_timeout 31000 +-- !user_privileges -- +'test_sys_tables'@'%' SELECT NO + -- !views -- -\N test_query_sys_db_1 test_view NONE NO root@% DEFINER utf8 \N +test_view diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy index 3292e7f83008c2..90b6e62c01954d 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -87,35 +87,13 @@ suite("test_query_sys_tables", "query,p0") { qt_columns("select TABLE_CATALOG, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, COLUMN_TYPE, COLUMN_SIZE from columns where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'") // test files - sql """ - CREATE FILE "test_file" - IN ${dbName1} - PROPERTIES - ( - "url" = "http://bit-images.bj.bcebos.com/bit2/index.html", - "catalog" = "test" - ); - """ - sql("use information_schema") - qt_files("select FILE_ID, FILE_NAME, FILE_TYPE from files where TABLE_SCHEMA = '${dbName1}'") - sql """ - DROP FILE "test_file" from test_query_sys_db_1 properties("catalog" = "test") - """ + // have no impl // test partitions - sql("use information_schema") - qt_partitions("select * from partitions") + // have no impl // test rowsets // have no tablet system table, add this later - // sql("use information_schema") - // qt_rowsets1("select count(*) from rowsets"); - // qt_rowsets2("select ROWSET_ID, TABLET_ID, DATA_DISK_SIZE, CREATION_TIME from rowsets") - - // test schema privileges - sql("use information_schema") - sql("GRANT LOAD_PRIV ON ctl.${dbName1}.* TO ROLE 'root'") - qt_schema_privileges("select * from schema_privileges where TABLE_SCHEMA = '${dbName1}'") // test schemata // create test dbs @@ -127,13 +105,7 @@ suite("test_query_sys_tables", "query,p0") { qt_schemata("select CATALOG_NAME, SCHEMA_NAME, SQL_PATH from schemata where SCHEMA_NAME = '${dbName1}' or SCHEMA_NAME = '${dbName2}' or SCHEMA_NAME = '${dbName3}'"); // test statistics - sql("use information_schema") - qt_statistics("select * from statistics") - - // test table privileges - sql("use information_schema") - sql("GRANT LOAD_PRIV ON ctl.${dbName1}.${tbName1} TO ROLE 'root'") - qt_table_privileges("select * from table_privileges where TABLE_NAME = '${tbName1}'") + // have no impl // test tables // create test dbs @@ -191,19 +163,25 @@ suite("test_query_sys_tables", "query,p0") { sql("use information_schema") qt_tables("select TABLE_CATALOG, TABLE_NAME, TABLE_TYPE, AVG_ROW_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH from tables where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'"); - // test user privileges - sql("use information_schema") - qt_user_privileges("select * from user_privileges") - // test variables // session_variables sql("use information_schema") sql("SET wait_timeout = 30000") qt_session_variables("select VARIABLE_NAME, VARIABLE_VALUE from session_variables where VARIABLE_NAME = 'wait_timeout'") + // global_variables sql("use information_schema") sql("SET GLOBAL wait_timeout = 31000") - qt_session_variables("select VARIABLE_NAME, VARIABLE_VALUE from global_variables where VARIABLE_NAME = 'wait_timeout'") + qt_global_variables("select VARIABLE_NAME, VARIABLE_VALUE from global_variables where VARIABLE_NAME = 'wait_timeout'") + + // test user_privileges + sql("CREATE USER 'test_sys_tables'") + sql("GRANT SELECT_PRIV ON *.*.* TO 'test_sys_tables'") + sql("use information_schema") + qt_user_privileges """ + select GRANTEE, PRIVILEGE_TYPE, IS_GRANTABLE from user_privileges where GRANTEE regexp '^\\'test' + """ + sql("DROP USER 'test_sys_tables'") // test views sql("use ${dbName1}") @@ -213,5 +191,5 @@ suite("test_query_sys_tables", "query,p0") { SELECT ccc as a FROM ${tbName1} """ sql("use information_schema") - qt_views("select * from views where TABLE_SCHEMA = '${dbName1}'") + qt_views("select TABLE_NAME, VIEW_DEFINITION from views where TABLE_SCHEMA = '${dbName1}'") } \ No newline at end of file From d1942e5b2b9c95104f0337f57b6d795ba5088616 Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Sat, 28 Jan 2023 11:38:14 +0800 Subject: [PATCH 19/24] fix format --- be/src/exec/schema_scanner/schema_collations_scanner.cpp | 6 ++---- .../exec/schema_scanner/schema_table_privileges_scanner.cpp | 2 +- .../exec/schema_scanner/schema_user_privileges_scanner.cpp | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.cpp b/be/src/exec/schema_scanner/schema_collations_scanner.cpp index 1f7ef6f12d7cda..6db2268bd4a5da 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_collations_scanner.cpp @@ -79,14 +79,12 @@ Status SchemaCollationsScanner::_fill_block_impl(vectorized::Block* block) { } // is_default for (int i = 0; i < row_num; ++i) { - StringRef str = - StringRef(_s_collations[i].is_default, strlen(_s_collations[i].is_default)); + StringRef str = StringRef(_s_collations[i].is_default, strlen(_s_collations[i].is_default)); fill_dest_column(block, &str, _tuple_desc->slots()[3]); } // IS_COMPILED for (int i = 0; i < row_num; ++i) { - StringRef str = - StringRef(_s_collations[i].is_compile, strlen(_s_collations[i].is_compile)); + StringRef str = StringRef(_s_collations[i].is_compile, strlen(_s_collations[i].is_compile)); fill_dest_column(block, &str, _tuple_desc->slots()[4]); } // sortlen diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp index 7290580cfa4fb2..4a94f6d8199b2c 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp @@ -132,7 +132,7 @@ Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block* block) for (int i = 0; i < privileges_num; ++i) { const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.privilege_type.c_str(), - priv_status.privilege_type.size()); + priv_status.privilege_type.size()); fill_dest_column(block, &str, _tuple_desc->slots()[4]); } } diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp index 4e2be50ca8de10..5865cc79a4b081 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp @@ -111,7 +111,7 @@ Status SchemaUserPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < privileges_num; ++i) { const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.privilege_type.c_str(), - priv_status.privilege_type.size()); + priv_status.privilege_type.size()); fill_dest_column(block, &str, _tuple_desc->slots()[2]); } } From 6978825aced2cda49397df155bda5cca497e7a4f Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Sat, 28 Jan 2023 23:29:29 +0800 Subject: [PATCH 20/24] fix rebase bug --- .../schema_scanner/schema_tables_scanner.cpp | 4 + be/src/vec/exec/vschema_scan_node.cpp | 31 ++- .../system/test_query_sys_tables.out | 46 ++++ .../system/test_query_sys_tables.groovy | 201 ++++++++++++++++++ .../system/test_query_sys_tables.groovy | 3 + 5 files changed, 281 insertions(+), 4 deletions(-) create mode 100644 regression-test/data/nereids_p0/system/test_query_sys_tables.out create mode 100644 regression-test/suites/nereids_p0/system/test_query_sys_tables.groovy diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index d9fd19f7e360c1..ec48bcf150b444 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -230,6 +230,8 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { time_slot.from_unixtime(create_time, TimezoneUtils::default_time_zone); fill_dest_column(block, &time_slot, _tuple_desc->slots()[14]); } + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[14]); } } // update_time @@ -244,6 +246,8 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { time_slot.from_unixtime(update_time, TimezoneUtils::default_time_zone); fill_dest_column(block, &time_slot, _tuple_desc->slots()[15]); } + } else { + fill_dest_column(block, nullptr, _tuple_desc->slots()[15]); } } // check_time diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index 28ce55787c9b99..2f76bed87f908e 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -261,34 +261,57 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, block->clear(); + std::vector index_map_inv(_src_tuple_desc->slots().size()); for (int i = 0; i < _slot_num; ++i) { auto dest_slot_desc = _dest_tuple_desc->slots()[i]; block->insert(ColumnWithTypeAndName(dest_slot_desc->get_empty_mutable_column(), dest_slot_desc->get_data_type_ptr(), dest_slot_desc->col_name())); + + // Map from index in column of schema table to slots. + index_map_inv[_index_map[i]] = i; } do { + vectorized::Block src_block; + + for (int i = 0; i < _src_tuple_desc->slots().size(); ++i) { + int j = index_map_inv[i]; + auto slot_desc = _dest_tuple_desc->slots()[j]; + src_block.insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), + slot_desc->get_data_type_ptr(), + slot_desc->col_name())); + } while (true) { RETURN_IF_CANCELLED(state); // get all slots from schema table. - RETURN_IF_ERROR(_schema_scanner->get_next_block(block, &schema_eos)); + RETURN_IF_ERROR(_schema_scanner->get_next_block(&src_block, &schema_eos)); if (schema_eos) { *eos = true; break; } - if (block->rows() >= state->batch_size()) { + if (src_block.rows() >= state->batch_size()) { break; } } - if (block->rows()) { + if (src_block.rows()) { + // block->check_number_of_rows(); + for (int i = 0; i < _slot_num; ++i) { + auto dest_slot_desc = _dest_tuple_desc->slots()[i]; + vectorized::MutableColumnPtr column_ptr = + std::move(*block->get_by_position(i).column).mutate(); + column_ptr->insert_range_from( + *src_block.get_by_name(dest_slot_desc->col_name()).column, 0, + src_block.rows()); + } RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, _dest_tuple_desc->slots().size())); - VLOG_ROW << "VSchemaScanNode output rows: " << block->rows(); + VLOG_ROW << "VSchemaScanNode output rows: " << src_block.rows(); + src_block.clear(); } } while (block->rows() == 0 && !(*eos)); diff --git a/regression-test/data/nereids_p0/system/test_query_sys_tables.out b/regression-test/data/nereids_p0/system/test_query_sys_tables.out new file mode 100644 index 00000000000000..6972d35ff6fcd3 --- /dev/null +++ b/regression-test/data/nereids_p0/system/test_query_sys_tables.out @@ -0,0 +1,46 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !backends -- +true + +-- !charsets -- +true + +-- !collations -- +true + +-- !columns -- +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 + +-- !schemata -- +internal test_query_sys_db_1 \N +internal test_query_sys_db_2 \N +internal test_query_sys_db_3 \N + +-- !tables -- +internal test_query_sys_tb_1 BASE TABLE 0 \N \N +internal test_query_sys_tb_2 BASE TABLE 0 \N \N +internal test_query_sys_tb_3 BASE TABLE 0 \N \N + +-- !session_variables -- +wait_timeout 30000 + +-- !global_variables -- +wait_timeout 31000 + +-- !user_privileges -- +'test_sys_tables'@'%' SELECT NO + +-- !views -- +test_view + diff --git a/regression-test/suites/nereids_p0/system/test_query_sys_tables.groovy b/regression-test/suites/nereids_p0/system/test_query_sys_tables.groovy new file mode 100644 index 00000000000000..52bfd989e61276 --- /dev/null +++ b/regression-test/suites/nereids_p0/system/test_query_sys_tables.groovy @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_query_sys_tables", "query,p0") { + sql "SET enable_nereids_planner=true" + sql "SET enable_vectorized_engine=true" + sql "SET enable_fallback_to_original_planner=false" + def dbName1 = "test_query_sys_db_1" + def dbName2 = "test_query_sys_db_2" + def dbName3 = "test_query_sys_db_3" + def tbName1 = "test_query_sys_tb_1" + def tbName2 = "test_query_sys_tb_2" + def tbName3 = "test_query_sys_tb_3" + sql("drop database IF EXISTS ${dbName1}") + sql("drop database IF EXISTS ${dbName2}") + sql("drop database IF EXISTS ${dbName3}") + + // test backends + sql("use information_schema") + qt_backends("select count(*) >= 1 from backends") + + // test charsets + sql("use information_schema") + qt_charsets("select count(*) >= 1 from character_sets") + + // test collations + sql("use information_schema") + qt_collations("select count(*) >= 1 from collations") + + // test columns + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName2}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName3}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use information_schema") + qt_columns("select TABLE_CATALOG, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, COLUMN_TYPE, COLUMN_SIZE from columns where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'") + + // test files + // have no impl + + // test partitions + // have no impl + + // test rowsets + // have no tablet system table, add this later + + // test schemata + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + + sql("use information_schema") + qt_schemata("select CATALOG_NAME, SCHEMA_NAME, SQL_PATH from schemata where SCHEMA_NAME = '${dbName1}' or SCHEMA_NAME = '${dbName2}' or SCHEMA_NAME = '${dbName3}'"); + + // test statistics + // have no impl + + // test tables + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName2}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName3}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + + sql("use information_schema") + qt_tables("select TABLE_CATALOG, TABLE_NAME, TABLE_TYPE, AVG_ROW_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH from tables where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'"); + + // test variables + // session_variables + sql("use information_schema") + sql("SET wait_timeout = 30000") + qt_session_variables("select VARIABLE_NAME, VARIABLE_VALUE from session_variables where VARIABLE_NAME = 'wait_timeout'") + + // global_variables + sql("use information_schema") + sql("SET GLOBAL wait_timeout = 31000") + qt_global_variables("select VARIABLE_NAME, VARIABLE_VALUE from global_variables where VARIABLE_NAME = 'wait_timeout'") + + // test user_privileges + sql("CREATE USER 'test_sys_tables'") + sql("GRANT SELECT_PRIV ON *.*.* TO 'test_sys_tables'") + sql("use information_schema") + qt_user_privileges """ + select GRANTEE, PRIVILEGE_TYPE, IS_GRANTABLE from user_privileges where GRANTEE regexp '^\\'test' + """ + sql("DROP USER 'test_sys_tables'") + + // test views + sql("use ${dbName1}") + sql """ + CREATE VIEW IF NOT EXISTS ${dbName1}.test_view (a) + AS + SELECT ccc as a FROM ${tbName1} + """ + sql("use information_schema") + qt_views("select TABLE_NAME, VIEW_DEFINITION from views where TABLE_SCHEMA = '${dbName1}'") +} \ No newline at end of file diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy index 90b6e62c01954d..0565abd3823153 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -22,6 +22,9 @@ suite("test_query_sys_tables", "query,p0") { def tbName1 = "test_query_sys_tb_1" def tbName2 = "test_query_sys_tb_2" def tbName3 = "test_query_sys_tb_3" + sql("drop database IF EXISTS ${dbName1}") + sql("drop database IF EXISTS ${dbName2}") + sql("drop database IF EXISTS ${dbName3}") // test backends sql("use information_schema") From 008dfda597b547d03fd6eddf45d0105871e315d7 Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Sun, 29 Jan 2023 13:14:36 +0800 Subject: [PATCH 21/24] change tuple_desc to column_desc --- be/src/exec/schema_scanner.cpp | 18 ++--- be/src/exec/schema_scanner.h | 2 +- .../schema_backends_scanner.cpp | 37 ++++++++-- .../schema_scanner/schema_backends_scanner.h | 1 + .../schema_charsets_scanner.cpp | 8 +-- .../schema_collations_scanner.cpp | 12 ++-- .../schema_scanner/schema_columns_scanner.cpp | 72 +++++++++---------- .../schema_scanner/schema_rowsets_scanner.cpp | 26 +++---- .../schema_schema_privileges_scanner.cpp | 10 +-- .../schema_schemata_scanner.cpp | 12 ++-- .../schema_table_privileges_scanner.cpp | 12 ++-- .../schema_scanner/schema_tables_scanner.cpp | 64 ++++++++--------- .../schema_user_privileges_scanner.cpp | 8 +-- .../schema_variables_scanner.cpp | 4 +- .../schema_scanner/schema_views_scanner.cpp | 20 +++--- 15 files changed, 165 insertions(+), 141 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 37eccaeefe49a2..ac7e28384b34d9 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -234,19 +234,15 @@ Status SchemaScanner::create_tuple_desc(ObjectPool* pool) { } Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, - const SlotDescriptor* slot_desc) { - if (!block->has(slot_desc->col_name()) || !slot_desc->is_materialized()) { + const ColumnDesc& col_desc) { + if (!block->has(col_desc.name)) { return Status::OK(); } vectorized::MutableColumnPtr column_ptr = - std::move(*block->get_by_name(slot_desc->col_name()).column).assume_mutable(); + std::move(*block->get_by_name(col_desc.name).column).assume_mutable(); vectorized::IColumn* col_ptr = column_ptr.get(); if (data == nullptr) { - if (!slot_desc->is_nullable()) { - return Status::InternalError("nonnull column contains NULL. column={}", - slot_desc->col_name()); - } auto* nullable_column = reinterpret_cast(col_ptr); nullable_column->insert_data(nullptr, 0); return Status::OK(); @@ -254,7 +250,7 @@ Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, auto* nullable_column = reinterpret_cast(col_ptr); nullable_column->get_null_map_data().push_back(0); col_ptr = &nullable_column->get_nested_column(); - switch (slot_desc->type().type) { + switch (col_desc.type) { case TYPE_HLL: { HyperLogLog* hll_slot = reinterpret_cast(data); reinterpret_cast(col_ptr)->get_data().emplace_back(*hll_slot); @@ -378,10 +374,10 @@ Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, } default: { - DCHECK(false) << "bad slot type: " << slot_desc->type(); + DCHECK(false) << "bad slot type: " << col_desc.type; std::stringstream ss; - ss << "Fail to convert schema type:'" << slot_desc->type() << " on column:`" - << slot_desc->col_name() + "`"; + ss << "Fail to convert schema type:'" << col_desc.type << " on column:`" + << std::string(col_desc.name) + "`"; return Status::InternalError(ss.str()); } } diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index 76059c52bd26e6..3a78e320659e00 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -96,7 +96,7 @@ class SchemaScanner { Status create_tuple_desc(ObjectPool* pool); Status create_columns(const std::vector* table_structure, ObjectPool* pool); - Status fill_dest_column(vectorized::Block* block, void* data, const SlotDescriptor* slot_desc); + Status fill_dest_column(vectorized::Block* block, void* data, const ColumnDesc& slot_desc); bool _is_init; // this is used for sub class diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.cpp b/be/src/exec/schema_scanner/schema_backends_scanner.cpp index 85b9527ce81bce..0d9a95f3b6ccdf 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_backends_scanner.cpp @@ -33,6 +33,33 @@ namespace doris { +SchemaScanner::ColumnDesc SchemaBackendsScanner::_s_tbls_columns[] = { + // name, type, size + {"BackendId", TYPE_BIGINT, sizeof(StringRef), false}, + {"TabletNum", TYPE_BIGINT, sizeof(StringRef), false}, + {"HeartbeatPort", TYPE_INT, sizeof(int), false}, + {"BePort", TYPE_INT, sizeof(int), false}, + {"HttpPort", TYPE_INT, sizeof(int), false}, + {"BrpcPort", TYPE_INT, sizeof(int), false}, + {"Cluster", TYPE_VARCHAR, sizeof(StringRef), false}, + {"IP", TYPE_VARCHAR, sizeof(StringRef), false}, + {"LastStartTime", TYPE_VARCHAR, sizeof(StringRef), false}, + {"LastHeartbeat", TYPE_VARCHAR, sizeof(StringRef), false}, + {"Alive", TYPE_VARCHAR, sizeof(StringRef), false}, + {"SystemDecommissioned", TYPE_VARCHAR, sizeof(StringRef), false}, + {"ClusterDecommissioned", TYPE_VARCHAR, sizeof(StringRef), false}, + {"DataUsedCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"AvailCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"TotalCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"UsedPct", TYPE_DOUBLE, sizeof(double), false}, + {"MaxDiskUsedPct", TYPE_DOUBLE, sizeof(double), false}, + {"RemoteUsedCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"Tag", TYPE_VARCHAR, sizeof(StringRef), false}, + {"ErrMsg", TYPE_VARCHAR, sizeof(StringRef), false}, + {"Version", TYPE_VARCHAR, sizeof(StringRef), false}, + {"Status", TYPE_VARCHAR, sizeof(StringRef), false}, +}; + SchemaBackendsScanner::SchemaBackendsScanner() : SchemaScanner(nullptr, 0, TSchemaTableType::SCH_BACKENDS) {} @@ -63,7 +90,7 @@ Status SchemaBackendsScanner::_fill_block_impl(vectorized::Block* block) { if (it == _col_name_to_type.end()) { if (_columns[col_idx].is_null) { for (int row_idx = 0; row_idx < row_num; ++row_idx) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[col_idx]); + fill_dest_column(block, nullptr, _s_tbls_columns[col_idx]); } } else { return Status::InternalError( @@ -73,22 +100,22 @@ Status SchemaBackendsScanner::_fill_block_impl(vectorized::Block* block) { } else if (it->second == TYPE_BIGINT) { for (int row_idx = 0; row_idx < row_num; ++row_idx) { fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].longVal, - _tuple_desc->slots()[col_idx]); + _s_tbls_columns[col_idx]); } } else if (it->second == TYPE_INT) { for (int row_idx = 0; row_idx < row_num; ++row_idx) { fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].intVal, - _tuple_desc->slots()[col_idx]); + _s_tbls_columns[col_idx]); } } else if (it->second == TYPE_VARCHAR) { for (int row_idx = 0; row_idx < row_num; ++row_idx) { fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].stringVal, - _tuple_desc->slots()[col_idx]); + _s_tbls_columns[col_idx]); } } else if (it->second == TYPE_DOUBLE) { for (int row_idx = 0; row_idx < row_num; ++row_idx) { fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].doubleVal, - _tuple_desc->slots()[col_idx]); + _s_tbls_columns[col_idx]); } } else { // other type diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.h b/be/src/exec/schema_scanner/schema_backends_scanner.h index 70b2483c16134b..21bc3348e97114 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.h +++ b/be/src/exec/schema_scanner/schema_backends_scanner.h @@ -36,6 +36,7 @@ class SchemaBackendsScanner : public SchemaScanner { // column_name -> type, set by _set_col_name_to_type() std::unordered_map _col_name_to_type; + static SchemaScanner::ColumnDesc _s_tbls_columns[]; std::vector _batch_data; }; diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp index 9ee6ac8f4da2ee..b4291290bce8dc 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp @@ -61,23 +61,23 @@ Status SchemaCharsetsScanner::_fill_block_impl(vectorized::Block* block) { // variables names for (int i = 0; i < row_num; ++i) { StringRef str = StringRef(_s_charsets[i].charset, strlen(_s_charsets[i].charset)); - fill_dest_column(block, &str, _tuple_desc->slots()[0]); + fill_dest_column(block, &str, _s_css_columns[0]); } // DEFAULT_COLLATE_NAME for (int i = 0; i < row_num; ++i) { StringRef str = StringRef(_s_charsets[i].default_collation, strlen(_s_charsets[i].default_collation)); - fill_dest_column(block, &str, _tuple_desc->slots()[1]); + fill_dest_column(block, &str, _s_css_columns[1]); } // DESCRIPTION for (int i = 0; i < row_num; ++i) { StringRef str = StringRef(_s_charsets[i].description, strlen(_s_charsets[i].description)); - fill_dest_column(block, &str, _tuple_desc->slots()[2]); + fill_dest_column(block, &str, _s_css_columns[2]); } // maxlen for (int i = 0; i < row_num; ++i) { int64_t src = _s_charsets[i].maxlen; - fill_dest_column(block, &src, _tuple_desc->slots()[3]); + fill_dest_column(block, &src, _s_css_columns[3]); } return Status::OK(); } diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.cpp b/be/src/exec/schema_scanner/schema_collations_scanner.cpp index 6db2268bd4a5da..85d73c1eebe481 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_collations_scanner.cpp @@ -65,32 +65,32 @@ Status SchemaCollationsScanner::_fill_block_impl(vectorized::Block* block) { // COLLATION_NAME for (int i = 0; i < row_num; ++i) { StringRef str = StringRef(_s_collations[i].name, strlen(_s_collations[i].name)); - fill_dest_column(block, &str, _tuple_desc->slots()[0]); + fill_dest_column(block, &str, _s_cols_columns[0]); } // charset for (int i = 0; i < row_num; ++i) { StringRef str = StringRef(_s_collations[i].charset, strlen(_s_collations[i].charset)); - fill_dest_column(block, &str, _tuple_desc->slots()[1]); + fill_dest_column(block, &str, _s_cols_columns[1]); } // id for (int i = 0; i < row_num; ++i) { int64_t src = _s_collations[i].id; - fill_dest_column(block, &src, _tuple_desc->slots()[2]); + fill_dest_column(block, &src, _s_cols_columns[2]); } // is_default for (int i = 0; i < row_num; ++i) { StringRef str = StringRef(_s_collations[i].is_default, strlen(_s_collations[i].is_default)); - fill_dest_column(block, &str, _tuple_desc->slots()[3]); + fill_dest_column(block, &str, _s_cols_columns[3]); } // IS_COMPILED for (int i = 0; i < row_num; ++i) { StringRef str = StringRef(_s_collations[i].is_compile, strlen(_s_collations[i].is_compile)); - fill_dest_column(block, &str, _tuple_desc->slots()[4]); + fill_dest_column(block, &str, _s_cols_columns[4]); } // sortlen for (int i = 0; i < row_num; ++i) { int64_t src = _s_collations[i].sortlen; - fill_dest_column(block, &src, _tuple_desc->slots()[5]); + fill_dest_column(block, &src, _s_cols_columns[5]); } return Status::OK(); } diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp index eeaf4110f9ee5b..036b291a32a41d 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp @@ -308,13 +308,13 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { { if (!_db_result.__isset.catalogs) { for (int i = 0; i < columns_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[0]); + fill_dest_column(block, nullptr, _s_col_columns[0]); } } else { std::string catalog_name = _db_result.catalogs[_db_index - 1]; StringRef str = StringRef(catalog_name.c_str(), catalog_name.size()); for (int i = 0; i < columns_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[0]); + fill_dest_column(block, &str, _s_col_columns[0]); } } } @@ -323,7 +323,7 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); StringRef str = StringRef(db_name.c_str(), db_name.size()); for (int i = 0; i < columns_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[1]); + fill_dest_column(block, &str, _s_col_columns[1]); } } // TABLE_NAME @@ -331,7 +331,7 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { StringRef str = StringRef(_table_result.tables[_table_index - 1].c_str(), _table_result.tables[_table_index - 1].length()); for (int i = 0; i < columns_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[2]); + fill_dest_column(block, &str, _s_col_columns[2]); } } // COLUMN_NAME @@ -339,20 +339,20 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < columns_num; ++i) { StringRef str = StringRef(_desc_result.columns[i].columnDesc.columnName.c_str(), _desc_result.columns[i].columnDesc.columnName.length()); - fill_dest_column(block, &str, _tuple_desc->slots()[3]); + fill_dest_column(block, &str, _s_col_columns[3]); } } // ORDINAL_POSITION { for (int i = 0; i < columns_num; ++i) { int64_t src = i + 1; - fill_dest_column(block, &src, _tuple_desc->slots()[4]); + fill_dest_column(block, &src, _s_col_columns[4]); } } // COLUMN_DEFAULT { for (int i = 0; i < columns_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[5]); + fill_dest_column(block, nullptr, _s_col_columns[5]); } } // IS_NULLABLE @@ -361,14 +361,14 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { if (_desc_result.columns[i].columnDesc.__isset.isAllowNull) { if (_desc_result.columns[i].columnDesc.isAllowNull) { StringRef str = StringRef("YES", 3); - fill_dest_column(block, &str, _tuple_desc->slots()[6]); + fill_dest_column(block, &str, _s_col_columns[6]); } else { StringRef str = StringRef("NO", 2); - fill_dest_column(block, &str, _tuple_desc->slots()[6]); + fill_dest_column(block, &str, _s_col_columns[6]); } } else { StringRef str = StringRef("NO", 2); - fill_dest_column(block, &str, _tuple_desc->slots()[6]); + fill_dest_column(block, &str, _s_col_columns[6]); } } } @@ -377,7 +377,7 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < columns_num; ++i) { std::string buffer = _to_mysql_data_type_string(_desc_result.columns[i].columnDesc); StringRef str = StringRef(buffer.c_str(), buffer.length()); - fill_dest_column(block, &str, _tuple_desc->slots()[7]); + fill_dest_column(block, &str, _s_col_columns[7]); } } // CHARACTER_MAXIMUM_LENGTH @@ -389,12 +389,12 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { data_type == TPrimitiveType::STRING) { if (_desc_result.columns[i].columnDesc.__isset.columnLength) { int64_t src = _desc_result.columns[i].columnDesc.columnLength; - fill_dest_column(block, &src, _tuple_desc->slots()[8]); + fill_dest_column(block, &src, _s_col_columns[8]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[8]); + fill_dest_column(block, nullptr, _s_col_columns[8]); } } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[8]); + fill_dest_column(block, nullptr, _s_col_columns[8]); } } } @@ -407,12 +407,12 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { data_type == TPrimitiveType::STRING) { if (_desc_result.columns[i].columnDesc.__isset.columnLength) { int64_t src = _desc_result.columns[i].columnDesc.columnLength * 4; - fill_dest_column(block, &src, _tuple_desc->slots()[9]); + fill_dest_column(block, &src, _s_col_columns[9]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[9]); + fill_dest_column(block, nullptr, _s_col_columns[9]); } } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[9]); + fill_dest_column(block, nullptr, _s_col_columns[9]); } } } @@ -421,9 +421,9 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < columns_num; ++i) { if (_desc_result.columns[i].columnDesc.__isset.columnPrecision) { int64_t src = _desc_result.columns[i].columnDesc.columnPrecision; - fill_dest_column(block, &src, _tuple_desc->slots()[10]); + fill_dest_column(block, &src, _s_col_columns[10]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[10]); + fill_dest_column(block, nullptr, _s_col_columns[10]); } } } @@ -432,28 +432,28 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < columns_num; ++i) { if (_desc_result.columns[i].columnDesc.__isset.columnScale) { int64_t src = _desc_result.columns[i].columnDesc.columnScale; - fill_dest_column(block, &src, _tuple_desc->slots()[11]); + fill_dest_column(block, &src, _s_col_columns[11]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[11]); + fill_dest_column(block, nullptr, _s_col_columns[11]); } } } // DATETIME_PRECISION { for (int i = 0; i < columns_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[12]); + fill_dest_column(block, nullptr, _s_col_columns[12]); } } // CHARACTER_SET_NAME { for (int i = 0; i < columns_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[13]); + fill_dest_column(block, nullptr, _s_col_columns[13]); } } // COLLATION_NAME { for (int i = 0; i < columns_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[14]); + fill_dest_column(block, nullptr, _s_col_columns[14]); } } // COLUMN_TYPE @@ -461,7 +461,7 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < columns_num; ++i) { std::string buffer = _type_to_string(_desc_result.columns[i].columnDesc); StringRef str = StringRef(buffer.c_str(), buffer.length()); - fill_dest_column(block, &str, _tuple_desc->slots()[15]); + fill_dest_column(block, &str, _s_col_columns[15]); } } // COLUMN_KEY @@ -470,10 +470,10 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { if (_desc_result.columns[i].columnDesc.__isset.columnKey) { StringRef str = StringRef(_desc_result.columns[i].columnDesc.columnKey.c_str(), _desc_result.columns[i].columnDesc.columnKey.length()); - fill_dest_column(block, &str, _tuple_desc->slots()[16]); + fill_dest_column(block, &str, _s_col_columns[16]); } else { StringRef str = StringRef("", 0); - fill_dest_column(block, &str, _tuple_desc->slots()[16]); + fill_dest_column(block, &str, _s_col_columns[16]); } } } @@ -481,14 +481,14 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { { for (int i = 0; i < columns_num; ++i) { StringRef str = StringRef("", 0); - fill_dest_column(block, &str, _tuple_desc->slots()[17]); + fill_dest_column(block, &str, _s_col_columns[17]); } } // PRIVILEGES { for (int i = 0; i < columns_num; ++i) { StringRef str = StringRef("", 0); - fill_dest_column(block, &str, _tuple_desc->slots()[18]); + fill_dest_column(block, &str, _s_col_columns[18]); } } // COLUMN_COMMENT @@ -496,7 +496,7 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < columns_num; ++i) { StringRef str = StringRef(_desc_result.columns[i].comment.c_str(), _desc_result.columns[i].comment.length()); - fill_dest_column(block, &str, _tuple_desc->slots()[19]); + fill_dest_column(block, &str, _s_col_columns[19]); } } // COLUMN_SIZE @@ -504,9 +504,9 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < columns_num; ++i) { if (_desc_result.columns[i].columnDesc.__isset.columnLength) { int64_t src = _desc_result.columns[i].columnDesc.columnLength; - fill_dest_column(block, &src, _tuple_desc->slots()[20]); + fill_dest_column(block, &src, _s_col_columns[20]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[20]); + fill_dest_column(block, nullptr, _s_col_columns[20]); } } } @@ -515,22 +515,22 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < columns_num; ++i) { if (_desc_result.columns[i].columnDesc.__isset.columnScale) { int64_t src = _desc_result.columns[i].columnDesc.columnScale; - fill_dest_column(block, &src, _tuple_desc->slots()[21]); + fill_dest_column(block, &src, _s_col_columns[21]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[21]); + fill_dest_column(block, nullptr, _s_col_columns[21]); } } } // GENERATION_EXPRESSION { for (int i = 0; i < columns_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[22]); + fill_dest_column(block, nullptr, _s_col_columns[22]); } } // SRS_ID { for (int i = 0; i < columns_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[23]); + fill_dest_column(block, nullptr, _s_col_columns[23]); } } return Status::OK(); diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp index 1110a7fe47d3a6..9e7e5e55e0198b 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp @@ -107,7 +107,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { { int64_t src = backend_id_; for (int i = fill_idx_begin; i < fill_idx_end; ++i) { - fill_dest_column(block, &src, _tuple_desc->slots()[0]); + fill_dest_column(block, &src, _s_tbls_columns[0]); } } // ROWSET_ID @@ -116,7 +116,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { RowsetSharedPtr rowset = rowsets_[i]; std::string rowset_id = rowset->rowset_id().to_string(); StringRef str = StringRef(rowset_id.c_str(), rowset_id.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[1]); + fill_dest_column(block, &str, _s_tbls_columns[1]); } } // TABLET_ID @@ -124,7 +124,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->rowset_meta()->tablet_id(); - fill_dest_column(block, &src, _tuple_desc->slots()[2]); + fill_dest_column(block, &src, _s_tbls_columns[2]); } } // ROWSET_NUM_ROWS @@ -132,7 +132,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->num_rows(); - fill_dest_column(block, &src, _tuple_desc->slots()[3]); + fill_dest_column(block, &src, _s_tbls_columns[3]); } } // TXN_ID @@ -140,7 +140,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->txn_id(); - fill_dest_column(block, &src, _tuple_desc->slots()[4]); + fill_dest_column(block, &src, _s_tbls_columns[4]); } } // NUM_SEGMENTS @@ -148,7 +148,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->num_segments(); - fill_dest_column(block, &src, _tuple_desc->slots()[5]); + fill_dest_column(block, &src, _s_tbls_columns[5]); } } // START_VERSION @@ -156,7 +156,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->start_version(); - fill_dest_column(block, &src, _tuple_desc->slots()[6]); + fill_dest_column(block, &src, _s_tbls_columns[6]); } } // END_VERSION @@ -164,7 +164,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; int64_t src = rowset->end_version(); - fill_dest_column(block, &src, _tuple_desc->slots()[7]); + fill_dest_column(block, &src, _s_tbls_columns[7]); } } // INDEX_DISK_SIZE @@ -172,7 +172,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; size_t src = rowset->index_disk_size(); - fill_dest_column(block, &src, _tuple_desc->slots()[8]); + fill_dest_column(block, &src, _s_tbls_columns[8]); } } // DATA_DISK_SIZE @@ -180,7 +180,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; size_t src = rowset->data_disk_size(); - fill_dest_column(block, &src, _tuple_desc->slots()[9]); + fill_dest_column(block, &src, _s_tbls_columns[9]); } } // CREATION_TIME @@ -188,7 +188,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; size_t src = rowset->creation_time(); - fill_dest_column(block, &src, _tuple_desc->slots()[10]); + fill_dest_column(block, &src, _s_tbls_columns[10]); } } // OLDEST_WRITE_TIMESTAMP @@ -196,7 +196,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; size_t src = rowset->oldest_write_timestamp(); - fill_dest_column(block, &src, _tuple_desc->slots()[11]); + fill_dest_column(block, &src, _s_tbls_columns[11]); } } // NEWEST_WRITE_TIMESTAMP @@ -204,7 +204,7 @@ Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { for (int i = fill_idx_begin; i < fill_idx_end; ++i) { RowsetSharedPtr rowset = rowsets_[i]; size_t src = rowset->newest_write_timestamp(); - fill_dest_column(block, &src, _tuple_desc->slots()[12]); + fill_dest_column(block, &src, _s_tbls_columns[12]); } } _rowsets_idx += fill_rowsets_num; diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp index cf70c414c45d8a..b34e59be50f270 100644 --- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp @@ -95,7 +95,7 @@ Status SchemaSchemaPrivilegesScanner::_fill_block_impl(vectorized::Block* block) for (int i = 0; i < privileges_num; ++i) { const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[0]); + fill_dest_column(block, &str, _s_tbls_columns[0]); } } // catalog @@ -104,7 +104,7 @@ Status SchemaSchemaPrivilegesScanner::_fill_block_impl(vectorized::Block* block) std::string definer = "def"; StringRef str = StringRef(definer.c_str(), definer.size()); for (int i = 0; i < privileges_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[1]); + fill_dest_column(block, &str, _s_tbls_columns[1]); } } // schema @@ -112,7 +112,7 @@ Status SchemaSchemaPrivilegesScanner::_fill_block_impl(vectorized::Block* block) for (int i = 0; i < privileges_num; ++i) { const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.schema.c_str(), priv_status.schema.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[2]); + fill_dest_column(block, &str, _s_tbls_columns[2]); } } // privilege type @@ -121,7 +121,7 @@ Status SchemaSchemaPrivilegesScanner::_fill_block_impl(vectorized::Block* block) const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.privilege_type.c_str(), priv_status.privilege_type.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[3]); + fill_dest_column(block, &str, _s_tbls_columns[3]); } } // is grantable @@ -130,7 +130,7 @@ Status SchemaSchemaPrivilegesScanner::_fill_block_impl(vectorized::Block* block) const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[4]); + fill_dest_column(block, &str, _s_tbls_columns[4]); } } return Status::OK(); diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp index d4ef60c2b69a78..381a5aeaa9341f 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp @@ -92,11 +92,11 @@ Status SchemaSchemataScanner::_fill_block_impl(vectorized::Block* block) { { for (int i = 0; i < dbs_num; ++i) { if (!_db_result.__isset.catalogs) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[0]); + fill_dest_column(block, nullptr, _s_columns[0]); } else { std::string catalog_name = _db_result.catalogs[i]; StringRef str = StringRef(catalog_name.c_str(), catalog_name.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[0]); + fill_dest_column(block, &str, _s_columns[0]); } } } @@ -105,7 +105,7 @@ Status SchemaSchemataScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < dbs_num; ++i) { std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[i]); StringRef str = StringRef(db_name.c_str(), db_name.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[1]); + fill_dest_column(block, &str, _s_columns[1]); } } // DEFAULT_CHARACTER_SET_NAME @@ -113,7 +113,7 @@ Status SchemaSchemataScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < dbs_num; ++i) { std::string src = "utf8"; StringRef str = StringRef(src.c_str(), src.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[2]); + fill_dest_column(block, &str, _s_columns[2]); } } // DEFAULT_COLLATION_NAME @@ -121,13 +121,13 @@ Status SchemaSchemataScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < dbs_num; ++i) { std::string src = "utf8_general_ci"; StringRef str = StringRef(src.c_str(), src.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[3]); + fill_dest_column(block, &str, _s_columns[3]); } } // SQL_PATH { for (int i = 0; i < dbs_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[4]); + fill_dest_column(block, nullptr, _s_columns[4]); } } return Status::OK(); diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp index 4a94f6d8199b2c..6a0de6d366430a 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp @@ -98,7 +98,7 @@ Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block* block) for (int i = 0; i < privileges_num; ++i) { const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[0]); + fill_dest_column(block, &str, _s_tbls_columns[0]); } } // catalog @@ -107,7 +107,7 @@ Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block* block) std::string definer = "def"; StringRef str = StringRef(definer.c_str(), definer.size()); for (int i = 0; i < privileges_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[1]); + fill_dest_column(block, &str, _s_tbls_columns[1]); } } // schema @@ -115,7 +115,7 @@ Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block* block) for (int i = 0; i < privileges_num; ++i) { const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.schema.c_str(), priv_status.schema.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[2]); + fill_dest_column(block, &str, _s_tbls_columns[2]); } } // table name @@ -124,7 +124,7 @@ Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block* block) const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.table_name.c_str(), priv_status.table_name.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[3]); + fill_dest_column(block, &str, _s_tbls_columns[3]); } } // privilege type @@ -133,7 +133,7 @@ Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block* block) const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.privilege_type.c_str(), priv_status.privilege_type.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[4]); + fill_dest_column(block, &str, _s_tbls_columns[4]); } } // is grantable @@ -142,7 +142,7 @@ Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block* block) const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[5]); + fill_dest_column(block, &str, _s_tbls_columns[5]); } } return Status::OK(); diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index ec48bcf150b444..307db7bf6faf02 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -126,11 +126,11 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { std::string catalog_name = _db_result.catalogs[_db_index - 1]; StringRef str_slot = StringRef(catalog_name.c_str(), catalog_name.size()); for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, &str_slot, _tuple_desc->slots()[0]); + fill_dest_column(block, &str_slot, _s_tbls_columns[0]); } } else { for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[0]); + fill_dest_column(block, nullptr, _s_tbls_columns[0]); } } // schema @@ -138,20 +138,20 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); StringRef str_slot = StringRef(db_name.c_str(), db_name.size()); for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, &str_slot, _tuple_desc->slots()[1]); + fill_dest_column(block, &str_slot, _s_tbls_columns[1]); } } // name for (int i = 0; i < table_num; ++i) { const std::string* src = &_table_result.tables[i].name; StringRef str_slot = StringRef(src->c_str(), src->size()); - fill_dest_column(block, &str_slot, _tuple_desc->slots()[2]); + fill_dest_column(block, &str_slot, _s_tbls_columns[2]); } // type for (int i = 0; i < table_num; ++i) { const std::string* src = &_table_result.tables[i].type; StringRef str_slot = StringRef(src->c_str(), src->size()); - fill_dest_column(block, &str_slot, _tuple_desc->slots()[3]); + fill_dest_column(block, &str_slot, _s_tbls_columns[3]); } // engine for (int i = 0; i < table_num; ++i) { @@ -159,27 +159,27 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { if (tbl_status.__isset.engine) { const std::string* src = &tbl_status.engine; StringRef str_slot = StringRef(src->c_str(), src->size()); - fill_dest_column(block, &str_slot, _tuple_desc->slots()[4]); + fill_dest_column(block, &str_slot, _s_tbls_columns[4]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[4]); + fill_dest_column(block, nullptr, _s_tbls_columns[4]); } } // version for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[5]); + fill_dest_column(block, nullptr, _s_tbls_columns[5]); } // row_format for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[6]); + fill_dest_column(block, nullptr, _s_tbls_columns[6]); } // rows for (int i = 0; i < table_num; ++i) { const TTableStatus& tbl_status = _table_result.tables[i]; if (tbl_status.__isset.rows) { int64_t src = tbl_status.rows; - fill_dest_column(block, &src, _tuple_desc->slots()[7]); + fill_dest_column(block, &src, _s_tbls_columns[7]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[7]); + fill_dest_column(block, nullptr, _s_tbls_columns[7]); } } // avg_row_length @@ -187,9 +187,9 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { const TTableStatus& tbl_status = _table_result.tables[i]; if (tbl_status.__isset.avg_row_length) { int64_t src = tbl_status.avg_row_length; - fill_dest_column(block, &src, _tuple_desc->slots()[8]); + fill_dest_column(block, &src, _s_tbls_columns[8]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[8]); + fill_dest_column(block, nullptr, _s_tbls_columns[8]); } } // data_length @@ -197,26 +197,26 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { const TTableStatus& tbl_status = _table_result.tables[i]; if (tbl_status.__isset.avg_row_length) { int64_t src = tbl_status.data_length; - fill_dest_column(block, &src, _tuple_desc->slots()[9]); + fill_dest_column(block, &src, _s_tbls_columns[9]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[9]); + fill_dest_column(block, nullptr, _s_tbls_columns[9]); } } // max_data_length for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[10]); + fill_dest_column(block, nullptr, _s_tbls_columns[10]); } // index_length for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[11]); + fill_dest_column(block, nullptr, _s_tbls_columns[11]); } // data_free for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[12]); + fill_dest_column(block, nullptr, _s_tbls_columns[12]); } // auto_increment for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[13]); + fill_dest_column(block, nullptr, _s_tbls_columns[13]); } // creation_time for (int i = 0; i < table_num; ++i) { @@ -224,14 +224,14 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { if (tbl_status.__isset.create_time) { int64_t create_time = tbl_status.create_time; if (create_time <= 0) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[14]); + fill_dest_column(block, nullptr, _s_tbls_columns[14]); } else { DateTimeValue time_slot; time_slot.from_unixtime(create_time, TimezoneUtils::default_time_zone); - fill_dest_column(block, &time_slot, _tuple_desc->slots()[14]); + fill_dest_column(block, &time_slot, _s_tbls_columns[14]); } } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[14]); + fill_dest_column(block, nullptr, _s_tbls_columns[14]); } } // update_time @@ -240,14 +240,14 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { if (tbl_status.__isset.update_time) { int64_t update_time = tbl_status.update_time; if (update_time <= 0) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[15]); + fill_dest_column(block, nullptr, _s_tbls_columns[15]); } else { DateTimeValue time_slot; time_slot.from_unixtime(update_time, TimezoneUtils::default_time_zone); - fill_dest_column(block, &time_slot, _tuple_desc->slots()[15]); + fill_dest_column(block, &time_slot, _s_tbls_columns[15]); } } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[15]); + fill_dest_column(block, nullptr, _s_tbls_columns[15]); } } // check_time @@ -256,11 +256,11 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { if (tbl_status.__isset.last_check_time) { int64_t check_time = tbl_status.last_check_time; if (check_time <= 0) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[16]); + fill_dest_column(block, nullptr, _s_tbls_columns[16]); } else { DateTimeValue time_slot; time_slot.from_unixtime(check_time, TimezoneUtils::default_time_zone); - fill_dest_column(block, &time_slot, _tuple_desc->slots()[16]); + fill_dest_column(block, &time_slot, _s_tbls_columns[16]); } } } @@ -270,25 +270,25 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { if (tbl_status.__isset.collation) { const std::string* src = &tbl_status.collation; StringRef str_slot = StringRef(src->c_str(), src->size()); - fill_dest_column(block, &str_slot, _tuple_desc->slots()[17]); + fill_dest_column(block, &str_slot, _s_tbls_columns[17]); } else { - fill_dest_column(block, nullptr, _tuple_desc->slots()[17]); + fill_dest_column(block, nullptr, _s_tbls_columns[17]); } } // checksum for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[18]); + fill_dest_column(block, nullptr, _s_tbls_columns[18]); } // create_options for (int i = 0; i < table_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[19]); + fill_dest_column(block, nullptr, _s_tbls_columns[19]); } // create_comment for (int i = 0; i < table_num; ++i) { const std::string* src = &_table_result.tables[i].comment; StringRef str_slot = StringRef(src->c_str(), src->size()); - fill_dest_column(block, &str_slot, _tuple_desc->slots()[20]); + fill_dest_column(block, &str_slot, _s_tbls_columns[20]); } return Status::OK(); } diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp index 5865cc79a4b081..cf791485294190 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp @@ -94,7 +94,7 @@ Status SchemaUserPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { for (int i = 0; i < privileges_num; ++i) { const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[0]); + fill_dest_column(block, &str, _s_tbls_columns[0]); } } // catalog @@ -103,7 +103,7 @@ Status SchemaUserPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { std::string definer = "def"; StringRef str = StringRef(definer.c_str(), definer.size()); for (int i = 0; i < privileges_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[1]); + fill_dest_column(block, &str, _s_tbls_columns[1]); } } // privilege type @@ -112,7 +112,7 @@ Status SchemaUserPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.privilege_type.c_str(), priv_status.privilege_type.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[2]); + fill_dest_column(block, &str, _s_tbls_columns[2]); } } // is grantable @@ -121,7 +121,7 @@ Status SchemaUserPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; StringRef str = StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[3]); + fill_dest_column(block, &str, _s_tbls_columns[3]); } } return Status::OK(); diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.cpp b/be/src/exec/schema_scanner/schema_variables_scanner.cpp index 9f84c0dd17fae9..ee45afe9bc4739 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_variables_scanner.cpp @@ -81,14 +81,14 @@ Status SchemaVariablesScanner::_fill_block_impl(vectorized::Block* block) { { for (auto& it : _var_result.variables) { StringRef str = StringRef(it.first.c_str(), it.first.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[0]); + fill_dest_column(block, &str, _s_vars_columns[0]); } } // value { for (auto& it : _var_result.variables) { StringRef str = StringRef(it.second.c_str(), it.second.size()); - fill_dest_column(block, &str, _tuple_desc->slots()[1]); + fill_dest_column(block, &str, _s_vars_columns[1]); } } return Status::OK(); diff --git a/be/src/exec/schema_scanner/schema_views_scanner.cpp b/be/src/exec/schema_scanner/schema_views_scanner.cpp index 568f871b6e9fdb..d72d82b22c6f14 100644 --- a/be/src/exec/schema_scanner/schema_views_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_views_scanner.cpp @@ -126,7 +126,7 @@ Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { // catalog { for (int i = 0; i < tables_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[0]); + fill_dest_column(block, nullptr, _s_tbls_columns[0]); } } // schema @@ -134,7 +134,7 @@ Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); StringRef str = StringRef(db_name.c_str(), db_name.size()); for (int i = 0; i < tables_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[1]); + fill_dest_column(block, &str, _s_tbls_columns[1]); } } // name @@ -143,7 +143,7 @@ Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { const TTableStatus& tbl_status = _table_result.tables[i]; const std::string* src = &tbl_status.name; StringRef str = StringRef(src->c_str(), src->size()); - fill_dest_column(block, &str, _tuple_desc->slots()[2]); + fill_dest_column(block, &str, _s_tbls_columns[2]); } } // definition @@ -152,7 +152,7 @@ Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { const TTableStatus& tbl_status = _table_result.tables[i]; const std::string* src = &tbl_status.ddl_sql; StringRef str = StringRef(src->c_str(), src->length()); - fill_dest_column(block, &str, _tuple_desc->slots()[3]); + fill_dest_column(block, &str, _s_tbls_columns[3]); } } // check_option @@ -160,7 +160,7 @@ Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { const std::string check_option = "NONE"; StringRef str = StringRef(check_option.c_str(), check_option.length()); for (int i = 0; i < tables_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[4]); + fill_dest_column(block, &str, _s_tbls_columns[4]); } } // is_updatable @@ -169,7 +169,7 @@ Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { const std::string is_updatable = "NO"; StringRef str = StringRef(is_updatable.c_str(), is_updatable.length()); for (int i = 0; i < tables_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[5]); + fill_dest_column(block, &str, _s_tbls_columns[5]); } } // definer @@ -178,7 +178,7 @@ Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { const std::string definer = "root@%"; StringRef str = StringRef(definer.c_str(), definer.length()); for (int i = 0; i < tables_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[6]); + fill_dest_column(block, &str, _s_tbls_columns[6]); } } // security_type @@ -187,7 +187,7 @@ Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { const std::string security_type = "DEFINER"; StringRef str = StringRef(security_type.c_str(), security_type.length()); for (int i = 0; i < tables_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[7]); + fill_dest_column(block, &str, _s_tbls_columns[7]); } } // character_set_client @@ -196,13 +196,13 @@ Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { const std::string encoding = "utf8"; StringRef str = StringRef(encoding.c_str(), encoding.length()); for (int i = 0; i < tables_num; ++i) { - fill_dest_column(block, &str, _tuple_desc->slots()[8]); + fill_dest_column(block, &str, _s_tbls_columns[8]); } } // collation_connection { for (int i = 0; i < tables_num; ++i) { - fill_dest_column(block, nullptr, _tuple_desc->slots()[9]); + fill_dest_column(block, nullptr, _s_tbls_columns[9]); } } return Status::OK(); From 908c179c57253e51f62b69201708c960ef73acaa Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Sun, 29 Jan 2023 17:29:02 +0800 Subject: [PATCH 22/24] refactor --- be/src/exec/schema_scanner.cpp | 115 +------- be/src/exec/schema_scanner.h | 20 +- .../schema_backends_scanner.cpp | 6 +- .../schema_scanner/schema_backends_scanner.h | 2 +- .../schema_charsets_scanner.cpp | 5 +- .../schema_scanner/schema_charsets_scanner.h | 2 +- .../schema_collations_scanner.cpp | 6 +- .../schema_collations_scanner.h | 2 +- .../schema_scanner/schema_columns_scanner.cpp | 5 +- .../schema_scanner/schema_columns_scanner.h | 2 +- .../schema_scanner/schema_dummy_scanner.cpp | 4 +- .../schema_scanner/schema_files_scanner.cpp | 6 +- .../schema_scanner/schema_files_scanner.h | 2 +- .../schema_partitions_scanner.cpp | 6 +- .../schema_partitions_scanner.h | 2 +- .../schema_scanner/schema_rowsets_scanner.cpp | 6 +- .../schema_scanner/schema_rowsets_scanner.h | 2 +- .../schema_schema_privileges_scanner.cpp | 6 +- .../schema_schema_privileges_scanner.h | 2 +- .../schema_schemata_scanner.cpp | 5 +- .../schema_scanner/schema_schemata_scanner.h | 2 +- .../schema_statistics_scanner.cpp | 6 +- .../schema_statistics_scanner.h | 2 +- .../schema_table_privileges_scanner.cpp | 6 +- .../schema_table_privileges_scanner.h | 2 +- .../schema_scanner/schema_tables_scanner.cpp | 7 +- .../schema_scanner/schema_tables_scanner.h | 2 +- .../schema_user_privileges_scanner.cpp | 6 +- .../schema_user_privileges_scanner.h | 2 +- .../schema_variables_scanner.cpp | 7 +- .../schema_scanner/schema_variables_scanner.h | 2 +- .../schema_scanner/schema_views_scanner.cpp | 7 +- .../schema_scanner/schema_views_scanner.h | 2 +- be/src/vec/exec/vschema_scan_node.cpp | 251 ++---------------- be/src/vec/exec/vschema_scan_node.h | 14 +- 35 files changed, 83 insertions(+), 441 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index ac7e28384b34d9..8e05320d89db0e 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -41,28 +41,16 @@ namespace doris { DorisServer* SchemaScanner::_s_doris_server; -SchemaScanner::SchemaScanner(ColumnDesc* columns, int column_num) +SchemaScanner::SchemaScanner(const std::vector& columns) : _is_init(false), _param(nullptr), _columns(columns), - _column_num(column_num), - _tuple_desc(nullptr), _schema_table_type(TSchemaTableType::SCH_INVALID) {} -SchemaScanner::SchemaScanner(ColumnDesc* columns, int column_num, TSchemaTableType::type type) - : _is_init(false), - _param(nullptr), - _columns(columns), - _column_num(column_num), - _tuple_desc(nullptr), - _schema_table_type(type) {} - -SchemaScanner::~SchemaScanner() { - if (_is_create_columns == true && _columns != nullptr) { - delete[] _columns; - _columns = nullptr; - } -} +SchemaScanner::SchemaScanner(const std::vector& columns, TSchemaTableType::type type) + : _is_init(false), _param(nullptr), _columns(columns), _schema_table_type(type) {} + +SchemaScanner::~SchemaScanner() {} Status SchemaScanner::start(RuntimeState* state) { if (!_is_init) { @@ -93,15 +81,10 @@ Status SchemaScanner::init(SchemaScannerParam* param, ObjectPool* pool) { return Status::InternalError("invalid parameter"); } - if (_schema_table_type == TSchemaTableType::SCH_BACKENDS) { - RETURN_IF_ERROR(create_columns(param->table_structure, pool)); - } - - if (nullptr == _columns) { + if (_columns.empty()) { return Status::InternalError("invalid parameter"); } - RETURN_IF_ERROR(create_tuple_desc(pool)); _param = param; _is_init = true; @@ -147,92 +130,6 @@ SchemaScanner* SchemaScanner::create(TSchemaTableType::type type) { } } -Status SchemaScanner::create_columns(const std::vector* table_structure, - ObjectPool* pool) { - _column_num = table_structure->size(); - _columns = new ColumnDesc[_column_num]; - _is_create_columns = true; - for (size_t idx = 0; idx < table_structure->size(); ++idx) { - _columns[idx].name = table_structure->at(idx).column_name.c_str(); - _columns[idx].type = thrift_to_type(table_structure->at(idx).type); - _columns[idx].size = table_structure->at(idx).len; - _columns[idx].is_null = table_structure->at(idx).is_null; - } - return Status::OK(); -} - -Status SchemaScanner::create_tuple_desc(ObjectPool* pool) { - int null_column = 0; - for (int i = 0; i < _column_num; ++i) { - if (_columns[i].is_null) { - null_column++; - } - } - - int offset = (null_column + 7) / 8; - std::vector slots; - int null_byte = 0; - int null_bit = 0; - - for (int i = 0; i < _column_num; ++i) { - TSlotDescriptor t_slot_desc; - if (_columns[i].type == TYPE_DECIMALV2) { - t_slot_desc.__set_slotType(TypeDescriptor::create_decimalv2_type(27, 9).to_thrift()); - } else { - TypeDescriptor descriptor(_columns[i].type); - if (_columns[i].precision >= 0 && _columns[i].scale >= 0) { - descriptor.precision = _columns[i].precision; - descriptor.scale = _columns[i].scale; - } - t_slot_desc.__set_slotType(descriptor.to_thrift()); - } - t_slot_desc.__set_colName(_columns[i].name); - t_slot_desc.__set_columnPos(i); - t_slot_desc.__set_byteOffset(offset); - - if (_columns[i].is_null) { - t_slot_desc.__set_nullIndicatorByte(null_byte); - t_slot_desc.__set_nullIndicatorBit(null_bit); - null_bit = (null_bit + 1) % 8; - - if (0 == null_bit) { - null_byte++; - } - } else { - t_slot_desc.__set_nullIndicatorByte(0); - t_slot_desc.__set_nullIndicatorBit(-1); - } - - t_slot_desc.id = i; - t_slot_desc.__set_slotIdx(i); - t_slot_desc.__set_isMaterialized(true); - - SlotDescriptor* slot = pool->add(new (std::nothrow) SlotDescriptor(t_slot_desc)); - - if (nullptr == slot) { - return Status::InternalError("no memory for _tuple_desc."); - } - - slots.push_back(slot); - offset += _columns[i].size; - } - - TTupleDescriptor t_tuple_desc; - t_tuple_desc.__set_byteSize(offset); - t_tuple_desc.__set_numNullBytes((null_byte * 8 + null_bit + 7) / 8); - _tuple_desc = pool->add(new (std::nothrow) TupleDescriptor(t_tuple_desc)); - - if (nullptr == _tuple_desc) { - return Status::InternalError("no memory for _tuple_desc."); - } - - for (int i = 0; i < slots.size(); ++i) { - _tuple_desc->add_slot(slots[i]); - } - - return Status::OK(); -} - Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, const ColumnDesc& col_desc) { if (!block->has(col_desc.name)) { diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index 3a78e320659e00..8e794792973416 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -75,8 +75,8 @@ class SchemaScanner { int precision = -1; int scale = -1; }; - SchemaScanner(ColumnDesc* columns, int column_num); - SchemaScanner(ColumnDesc* columns, int column_num, TSchemaTableType::type type); + SchemaScanner(const std::vector& columns); + SchemaScanner(const std::vector& columns, TSchemaTableType::type type); virtual ~SchemaScanner(); // init object need information, schema etc. @@ -84,32 +84,22 @@ class SchemaScanner { // Start to work virtual Status start(RuntimeState* state); virtual Status get_next_block(vectorized::Block* block, bool* eos); + const std::vector& get_column_desc() const { return _columns; } // factory function static SchemaScanner* create(TSchemaTableType::type type); - const TupleDescriptor* tuple_desc() const { return _tuple_desc; } const TSchemaTableType::type type() const { return _schema_table_type; } static void set_doris_server(DorisServer* doris_server) { _s_doris_server = doris_server; } protected: - Status create_tuple_desc(ObjectPool* pool); - Status create_columns(const std::vector* table_structure, - ObjectPool* pool); Status fill_dest_column(vectorized::Block* block, void* data, const ColumnDesc& slot_desc); bool _is_init; // this is used for sub class SchemaScannerParam* _param; - // pointer to schema table's column desc - ColumnDesc* _columns; - // num of columns - int _column_num; - TupleDescriptor* _tuple_desc; - - // _is_create_columns means if ColumnDesc is created from FE. - // `_columns` should be deleted if _is_create_columns = true. - bool _is_create_columns = false; + // schema table's column desc + std::vector _columns; static DorisServer* _s_doris_server; diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.cpp b/be/src/exec/schema_scanner/schema_backends_scanner.cpp index 0d9a95f3b6ccdf..e9c008f8a23290 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_backends_scanner.cpp @@ -33,7 +33,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaBackendsScanner::_s_tbls_columns[] = { +std::vector SchemaBackendsScanner::_s_tbls_columns = { // name, type, size {"BackendId", TYPE_BIGINT, sizeof(StringRef), false}, {"TabletNum", TYPE_BIGINT, sizeof(StringRef), false}, @@ -61,7 +61,7 @@ SchemaScanner::ColumnDesc SchemaBackendsScanner::_s_tbls_columns[] = { }; SchemaBackendsScanner::SchemaBackendsScanner() - : SchemaScanner(nullptr, 0, TSchemaTableType::SCH_BACKENDS) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_BACKENDS) {} Status SchemaBackendsScanner::start(RuntimeState* state) { if (!_is_init) { @@ -85,7 +85,7 @@ Status SchemaBackendsScanner::get_next_block(vectorized::Block* block, bool* eos Status SchemaBackendsScanner::_fill_block_impl(vectorized::Block* block) { auto row_num = _batch_data.size(); - for (size_t col_idx = 0; col_idx < _column_num; ++col_idx) { + for (size_t col_idx = 0; col_idx < _columns.size(); ++col_idx) { auto it = _col_name_to_type.find(_columns[col_idx].name); if (it == _col_name_to_type.end()) { if (_columns[col_idx].is_null) { diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.h b/be/src/exec/schema_scanner/schema_backends_scanner.h index 21bc3348e97114..82ca121561c5f8 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.h +++ b/be/src/exec/schema_scanner/schema_backends_scanner.h @@ -36,7 +36,7 @@ class SchemaBackendsScanner : public SchemaScanner { // column_name -> type, set by _set_col_name_to_type() std::unordered_map _col_name_to_type; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; std::vector _batch_data; }; diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp index b4291290bce8dc..56b446fa56878a 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp @@ -22,7 +22,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaCharsetsScanner::_s_css_columns[] = { +std::vector SchemaCharsetsScanner::_s_css_columns = { // name, type, size {"CHARACTER_SET_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, {"DEFAULT_COLLATE_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -36,8 +36,7 @@ SchemaCharsetsScanner::CharsetStruct SchemaCharsetsScanner::_s_charsets[] = { }; SchemaCharsetsScanner::SchemaCharsetsScanner() - : SchemaScanner(_s_css_columns, sizeof(_s_css_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_CHARSETS) {} + : SchemaScanner(_s_css_columns, TSchemaTableType::SCH_CHARSETS) {} SchemaCharsetsScanner::~SchemaCharsetsScanner() {} diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.h b/be/src/exec/schema_scanner/schema_charsets_scanner.h index eb5f61bfc63421..37b2866bab31e3 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.h @@ -40,7 +40,7 @@ class SchemaCharsetsScanner : public SchemaScanner { Status _fill_block_impl(vectorized::Block* block); - static SchemaScanner::ColumnDesc _s_css_columns[]; + static std::vector _s_css_columns; static CharsetStruct _s_charsets[]; }; diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.cpp b/be/src/exec/schema_scanner/schema_collations_scanner.cpp index 85d73c1eebe481..eb35c30f2c5a4e 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_collations_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaCollationsScanner::_s_cols_columns[] = { +std::vector SchemaCollationsScanner::_s_cols_columns = { // name, type, size {"COLLATION_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, {"CHARACTER_SET_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -39,9 +39,7 @@ SchemaCollationsScanner::CollationStruct SchemaCollationsScanner::_s_collations[ }; SchemaCollationsScanner::SchemaCollationsScanner() - : SchemaScanner(_s_cols_columns, - sizeof(_s_cols_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_COLLATIONS) {} + : SchemaScanner(_s_cols_columns, TSchemaTableType::SCH_COLLATIONS) {} SchemaCollationsScanner::~SchemaCollationsScanner() {} diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.h b/be/src/exec/schema_scanner/schema_collations_scanner.h index 58f0ee54de4901..d421cf3818365f 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.h +++ b/be/src/exec/schema_scanner/schema_collations_scanner.h @@ -42,7 +42,7 @@ class SchemaCollationsScanner : public SchemaScanner { Status _fill_block_impl(vectorized::Block* block); - static SchemaScanner::ColumnDesc _s_cols_columns[]; + static std::vector _s_cols_columns; static CollationStruct _s_collations[]; }; diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp index 036b291a32a41d..8344a67484a847 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp @@ -27,7 +27,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaColumnsScanner::_s_col_columns[] = { +std::vector SchemaColumnsScanner::_s_col_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -56,8 +56,7 @@ SchemaScanner::ColumnDesc SchemaColumnsScanner::_s_col_columns[] = { }; SchemaColumnsScanner::SchemaColumnsScanner() - : SchemaScanner(_s_col_columns, sizeof(_s_col_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_COLUMNS), + : SchemaScanner(_s_col_columns, TSchemaTableType::SCH_COLUMNS), _db_index(0), _table_index(0) {} diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.h b/be/src/exec/schema_scanner/schema_columns_scanner.h index 6f50bfd58510d7..9ebe10b380d000 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.h +++ b/be/src/exec/schema_scanner/schema_columns_scanner.h @@ -44,7 +44,7 @@ class SchemaColumnsScanner : public SchemaScanner { TGetDbsResult _db_result; TGetTablesResult _table_result; TDescribeTableResult _desc_result; - static SchemaScanner::ColumnDesc _s_col_columns[]; + static std::vector _s_col_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp index f195d0dfc75635..b85885571837e1 100644 --- a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp @@ -18,12 +18,12 @@ #include "schema_dummy_scanner.h" namespace { -doris::SchemaScanner::ColumnDesc DUMMY_COLUMN; +std::vector DUMMY_COLUMN; } namespace doris { -SchemaDummyScanner::SchemaDummyScanner() : SchemaScanner(&DUMMY_COLUMN, 0) {} +SchemaDummyScanner::SchemaDummyScanner() : SchemaScanner(DUMMY_COLUMN) {} SchemaDummyScanner::~SchemaDummyScanner() {} diff --git a/be/src/exec/schema_scanner/schema_files_scanner.cpp b/be/src/exec/schema_scanner/schema_files_scanner.cpp index a0cda249dac01d..762db774f96eb5 100644 --- a/be/src/exec/schema_scanner/schema_files_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_files_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaFilesScanner::_s_tbls_columns[] = { +std::vector SchemaFilesScanner::_s_tbls_columns = { // name, type, size, is_null {"FILE_ID", TYPE_BIGINT, sizeof(int64_t), true}, {"FILE_NAME", TYPE_STRING, sizeof(StringRef), true}, @@ -66,9 +66,7 @@ SchemaScanner::ColumnDesc SchemaFilesScanner::_s_tbls_columns[] = { }; SchemaFilesScanner::SchemaFilesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_FILES), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_FILES), _db_index(0), _table_index(0) {} diff --git a/be/src/exec/schema_scanner/schema_files_scanner.h b/be/src/exec/schema_scanner/schema_files_scanner.h index 834b98908ae13c..0e33bf6dd32e00 100644 --- a/be/src/exec/schema_scanner/schema_files_scanner.h +++ b/be/src/exec/schema_scanner/schema_files_scanner.h @@ -34,7 +34,7 @@ class SchemaFilesScanner : public SchemaScanner { int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp index 6b268e6563fe2f..41ba26da4b6e23 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp @@ -24,7 +24,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaPartitionsScanner::_s_tbls_columns[] = { +std::vector SchemaPartitionsScanner::_s_tbls_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), true}, @@ -54,9 +54,7 @@ SchemaScanner::ColumnDesc SchemaPartitionsScanner::_s_tbls_columns[] = { }; SchemaPartitionsScanner::SchemaPartitionsScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_PARTITIONS), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_PARTITIONS), _db_index(0), _table_index(0) {} diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.h b/be/src/exec/schema_scanner/schema_partitions_scanner.h index 6fe1baf480deb1..197f917a1de4f1 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.h +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.h @@ -34,7 +34,7 @@ class SchemaPartitionsScanner : public SchemaScanner { int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp index 9e7e5e55e0198b..2744531bd775f6 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp @@ -31,7 +31,7 @@ #include "runtime/primitive_type.h" #include "vec/common/string_ref.h" namespace doris { -SchemaScanner::ColumnDesc SchemaRowsetsScanner::_s_tbls_columns[] = { +std::vector SchemaRowsetsScanner::_s_tbls_columns = { // name, type, size, is_null {"BACKEND_ID", TYPE_BIGINT, sizeof(int64_t), true}, {"ROWSET_ID", TYPE_VARCHAR, sizeof(StringRef), true}, @@ -50,9 +50,7 @@ SchemaScanner::ColumnDesc SchemaRowsetsScanner::_s_tbls_columns[] = { }; SchemaRowsetsScanner::SchemaRowsetsScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_ROWSETS), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_ROWSETS), backend_id_(0), _rowsets_idx(0) {}; diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.h b/be/src/exec/schema_scanner/schema_rowsets_scanner.h index 273abe3a950332..5869f97cdcd655 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.h @@ -40,7 +40,7 @@ class SchemaRowsetsScanner : public SchemaScanner { Status _get_all_rowsets(); Status _fill_block_impl(vectorized::Block* block); - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; int64_t backend_id_ = 0; size_t _rowsets_idx = 0; std::vector rowsets_; diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp index b34e59be50f270..03072073d24d10 100644 --- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaSchemaPrivilegesScanner::_s_tbls_columns[] = { +std::vector SchemaSchemaPrivilegesScanner::_s_tbls_columns = { // name, type, size, is_null {"GRANTEE", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, @@ -33,9 +33,7 @@ SchemaScanner::ColumnDesc SchemaSchemaPrivilegesScanner::_s_tbls_columns[] = { }; SchemaSchemaPrivilegesScanner::SchemaSchemaPrivilegesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_SCHEMA_PRIVILEGES) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_SCHEMA_PRIVILEGES) {} SchemaSchemaPrivilegesScanner::~SchemaSchemaPrivilegesScanner() {} diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h index 3222e9d81ac129..a52643afd12fb1 100644 --- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h @@ -35,7 +35,7 @@ class SchemaSchemaPrivilegesScanner : public SchemaScanner { Status _fill_block_impl(vectorized::Block* block); TListPrivilegesResult _priv_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp index 381a5aeaa9341f..647a19a47aa2ed 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaSchemataScanner::_s_columns[] = { +std::vector SchemaSchemataScanner::_s_columns = { // name, type, size {"CATALOG_NAME", TYPE_VARCHAR, sizeof(StringRef), true}, {"SCHEMA_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -33,8 +33,7 @@ SchemaScanner::ColumnDesc SchemaSchemataScanner::_s_columns[] = { }; SchemaSchemataScanner::SchemaSchemataScanner() - : SchemaScanner(_s_columns, sizeof(_s_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_SCHEMATA) {} + : SchemaScanner(_s_columns, TSchemaTableType::SCH_SCHEMATA) {} SchemaSchemataScanner::~SchemaSchemataScanner() = default; diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.h b/be/src/exec/schema_scanner/schema_schemata_scanner.h index 0077520441dd02..572f8b741df23f 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.h +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.h @@ -34,7 +34,7 @@ class SchemaSchemataScanner : public SchemaScanner { Status _fill_block_impl(vectorized::Block* block); TGetDbsResult _db_result; - static SchemaScanner::ColumnDesc _s_columns[]; + static std::vector _s_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_statistics_scanner.cpp b/be/src/exec/schema_scanner/schema_statistics_scanner.cpp index 366765dda604f3..ed2e713da03aed 100644 --- a/be/src/exec/schema_scanner/schema_statistics_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_statistics_scanner.cpp @@ -22,7 +22,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaStatisticsScanner::_s_cols_statistics[] = { +std::vector SchemaStatisticsScanner::_s_cols_statistics = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -41,9 +41,7 @@ SchemaScanner::ColumnDesc SchemaStatisticsScanner::_s_cols_statistics[] = { {"COMMENT", TYPE_VARCHAR, sizeof(StringRef), true}, }; -SchemaStatisticsScanner::SchemaStatisticsScanner() - : SchemaScanner(_s_cols_statistics, - sizeof(_s_cols_statistics) / sizeof(SchemaScanner::ColumnDesc)) {} +SchemaStatisticsScanner::SchemaStatisticsScanner() : SchemaScanner(_s_cols_statistics) {} SchemaStatisticsScanner::~SchemaStatisticsScanner() {} diff --git a/be/src/exec/schema_scanner/schema_statistics_scanner.h b/be/src/exec/schema_scanner/schema_statistics_scanner.h index ebbcc27216fead..e31c27277b2de2 100644 --- a/be/src/exec/schema_scanner/schema_statistics_scanner.h +++ b/be/src/exec/schema_scanner/schema_statistics_scanner.h @@ -26,6 +26,6 @@ class SchemaStatisticsScanner : public SchemaScanner { ~SchemaStatisticsScanner() override; private: - static SchemaScanner::ColumnDesc _s_cols_statistics[]; + static std::vector _s_cols_statistics; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp index 6a0de6d366430a..d959145168c66c 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp @@ -25,7 +25,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaTablePrivilegesScanner::_s_tbls_columns[] = { +std::vector SchemaTablePrivilegesScanner::_s_tbls_columns = { // name, type, size, is_null {"GRANTEE", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, @@ -36,9 +36,7 @@ SchemaScanner::ColumnDesc SchemaTablePrivilegesScanner::_s_tbls_columns[] = { }; SchemaTablePrivilegesScanner::SchemaTablePrivilegesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_TABLE_PRIVILEGES) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_TABLE_PRIVILEGES) {} SchemaTablePrivilegesScanner::~SchemaTablePrivilegesScanner() {} diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h index 82629522b8c2fc..00545717c22ea0 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h @@ -35,7 +35,7 @@ class SchemaTablePrivilegesScanner : public SchemaScanner { Status _fill_block_impl(vectorized::Block* block); TListPrivilegesResult _priv_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index 307db7bf6faf02..054897b516b202 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -25,7 +25,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaTablesScanner::_s_tbls_columns[] = { +std::vector SchemaTablesScanner::_s_tbls_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -51,10 +51,7 @@ SchemaScanner::ColumnDesc SchemaTablesScanner::_s_tbls_columns[] = { }; SchemaTablesScanner::SchemaTablesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_TABLES), - _db_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_TABLES), _db_index(0) {} SchemaTablesScanner::~SchemaTablesScanner() {} diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h index 933d1f8e9a11a3..18488451ccf819 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_tables_scanner.h @@ -39,7 +39,7 @@ class SchemaTablesScanner : public SchemaScanner { int _db_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp index cf791485294190..4fcdfecfbe8008 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaUserPrivilegesScanner::_s_tbls_columns[] = { +std::vector SchemaUserPrivilegesScanner::_s_tbls_columns = { // name, type, size, is_null {"GRANTEE", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, @@ -32,9 +32,7 @@ SchemaScanner::ColumnDesc SchemaUserPrivilegesScanner::_s_tbls_columns[] = { }; SchemaUserPrivilegesScanner::SchemaUserPrivilegesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_USER_PRIVILEGES) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_USER_PRIVILEGES) {} SchemaUserPrivilegesScanner::~SchemaUserPrivilegesScanner() {} diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h index a4b5313be51f78..1bef2b340b9e10 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h @@ -35,7 +35,7 @@ class SchemaUserPrivilegesScanner : public SchemaScanner { Status _fill_block_impl(vectorized::Block* block); TListPrivilegesResult _priv_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.cpp b/be/src/exec/schema_scanner/schema_variables_scanner.cpp index ee45afe9bc4739..38fa4e0af191af 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_variables_scanner.cpp @@ -24,17 +24,14 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaVariablesScanner::_s_vars_columns[] = { +std::vector SchemaVariablesScanner::_s_vars_columns = { // name, type, size {"VARIABLE_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, {"VARIABLE_VALUE", TYPE_VARCHAR, sizeof(StringRef), false}, }; SchemaVariablesScanner::SchemaVariablesScanner(TVarType::type type) - : SchemaScanner(_s_vars_columns, - sizeof(_s_vars_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_VARIABLES), - _type(type) {} + : SchemaScanner(_s_vars_columns, TSchemaTableType::SCH_VARIABLES), _type(type) {} SchemaVariablesScanner::~SchemaVariablesScanner() {} diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.h b/be/src/exec/schema_scanner/schema_variables_scanner.h index 257858684063f4..4befea7bda4803 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.h +++ b/be/src/exec/schema_scanner/schema_variables_scanner.h @@ -41,7 +41,7 @@ class SchemaVariablesScanner : public SchemaScanner { Status _fill_block_impl(vectorized::Block* block); - static SchemaScanner::ColumnDesc _s_vars_columns[]; + static std::vector _s_vars_columns; TShowVariableResult _var_result; TVarType::type _type; diff --git a/be/src/exec/schema_scanner/schema_views_scanner.cpp b/be/src/exec/schema_scanner/schema_views_scanner.cpp index d72d82b22c6f14..fb64337cce70d4 100644 --- a/be/src/exec/schema_scanner/schema_views_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_views_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaViewsScanner::_s_tbls_columns[] = { +std::vector SchemaViewsScanner::_s_tbls_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -38,10 +38,7 @@ SchemaScanner::ColumnDesc SchemaViewsScanner::_s_tbls_columns[] = { }; SchemaViewsScanner::SchemaViewsScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc), - TSchemaTableType::SCH_VIEWS), - _db_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_VIEWS), _db_index(0) {} SchemaViewsScanner::~SchemaViewsScanner() {} diff --git a/be/src/exec/schema_scanner/schema_views_scanner.h b/be/src/exec/schema_scanner/schema_views_scanner.h index 968f99ca299fa4..9eeca601f32e05 100644 --- a/be/src/exec/schema_scanner/schema_views_scanner.h +++ b/be/src/exec/schema_scanner/schema_views_scanner.h @@ -37,7 +37,7 @@ class SchemaViewsScanner : public SchemaScanner { int _db_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index 2f76bed87f908e..7d02863f7b7cbd 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -17,16 +17,21 @@ #include "vec/exec/vschema_scan_node.h" +#include +#include + #include "common/status.h" #include "exec/text_converter.h" #include "exec/text_converter.hpp" #include "gen_cpp/PlanNodes_types.h" +#include "runtime/descriptors.h" #include "runtime/runtime_state.h" #include "util/runtime_profile.h" #include "util/types.h" #include "vec/columns/column.h" #include "vec/common/string_ref.h" #include "vec/core/types.h" +#include "vec/data_types/data_type_factory.hpp" namespace doris::vectorized { VSchemaScanNode::VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, @@ -35,26 +40,13 @@ VSchemaScanNode::VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, _is_init(false), _table_name(tnode.schema_scan_node.table_name), _tuple_id(tnode.schema_scan_node.tuple_id), - _src_tuple_desc(nullptr), _dest_tuple_desc(nullptr), _tuple_idx(0), _slot_num(0), _tuple_pool(nullptr), - _schema_scanner(nullptr), - _src_tuple(nullptr), - _src_single_tuple(nullptr), - _dest_single_tuple(nullptr) {} - -VSchemaScanNode::~VSchemaScanNode() { - delete[] reinterpret_cast(_src_tuple); - _src_tuple = nullptr; + _schema_scanner(nullptr) {} - delete[] reinterpret_cast(_src_single_tuple); - _src_single_tuple = nullptr; - - delete[] reinterpret_cast(_dest_single_tuple); - _dest_single_tuple = nullptr; -} +VSchemaScanNode::~VSchemaScanNode() {} Status VSchemaScanNode::init(const TPlanNode& tnode, RuntimeState* state) { RETURN_IF_ERROR(ExecNode::init(tnode, state)); @@ -177,71 +169,42 @@ Status VSchemaScanNode::prepare(RuntimeState* state) { } RETURN_IF_ERROR(_schema_scanner->init(&_scanner_param, _pool)); - // get column info from scanner - _src_tuple_desc = _schema_scanner->tuple_desc(); - - if (nullptr == _src_tuple_desc) { - return Status::InternalError("failed to get src schema tuple desc."); - } - - _src_tuple = - reinterpret_cast(new (std::nothrow) char[_src_tuple_desc->byte_size()]); - - if (nullptr == _src_tuple) { - return Status::InternalError("new src tuple failed."); - } + const std::vector& columns_desc(_schema_scanner->get_column_desc()); - // if src tuple desc slots is zero, it's the dummy slots. - if (0 == _src_tuple_desc->slots().size()) { + // if src columns size is zero, it's the dummy slots. + if (0 == columns_desc.size()) { _slot_num = 0; } // check if type is ok. - if (_slot_num > 0) { - _index_map.resize(_slot_num); - } for (int i = 0; i < _slot_num; ++i) { // TODO(zhaochun): Is this slow? int j = 0; - for (; j < _src_tuple_desc->slots().size(); ++j) { - if (boost::iequals(_dest_tuple_desc->slots()[i]->col_name(), - _src_tuple_desc->slots()[j]->col_name())) { + for (; j < columns_desc.size(); ++j) { + if (boost::iequals(_dest_tuple_desc->slots()[i]->col_name(), columns_desc[j].name)) { break; } } - if (j >= _src_tuple_desc->slots().size()) { + if (j >= columns_desc.size()) { LOG(WARNING) << "no match column for this column(" << _dest_tuple_desc->slots()[i]->col_name() << ")"; return Status::InternalError("no match column for this column."); } - if (_src_tuple_desc->slots()[j]->type().type != _dest_tuple_desc->slots()[i]->type().type) { - LOG(WARNING) << "schema not match. input is " << _src_tuple_desc->slots()[j]->col_name() - << "(" << _src_tuple_desc->slots()[j]->type() << ") and output is " + if (columns_desc[j].type != _dest_tuple_desc->slots()[i]->type().type) { + LOG(WARNING) << "schema not match. input is " << columns_desc[j].name << "(" + << columns_desc[j].type << ") and output is " << _dest_tuple_desc->slots()[i]->col_name() << "(" << _dest_tuple_desc->slots()[i]->type() << ")"; return Status::InternalError("schema not match."); } - _index_map[i] = j; } // TODO(marcel): add int _tuple_idx indexed by TupleId somewhere in runtime_state.h _tuple_idx = 0; _is_init = true; - _src_single_tuple = - reinterpret_cast(new (std::nothrow) char[_src_tuple_desc->byte_size()]); - if (nullptr == _src_single_tuple) { - return Status::InternalError("new src single tuple failed."); - } - - _dest_single_tuple = - reinterpret_cast(new (std::nothrow) char[_dest_tuple_desc->byte_size()]); - if (nullptr == _dest_single_tuple) { - return Status::InternalError("new desc single tuple failed."); - } - return Status::OK(); } @@ -260,27 +223,24 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, bool schema_eos = false; block->clear(); + const std::vector& columns_desc(_schema_scanner->get_column_desc()); - std::vector index_map_inv(_src_tuple_desc->slots().size()); for (int i = 0; i < _slot_num; ++i) { auto dest_slot_desc = _dest_tuple_desc->slots()[i]; block->insert(ColumnWithTypeAndName(dest_slot_desc->get_empty_mutable_column(), dest_slot_desc->get_data_type_ptr(), dest_slot_desc->col_name())); - - // Map from index in column of schema table to slots. - index_map_inv[_index_map[i]] = i; } do { vectorized::Block src_block; - for (int i = 0; i < _src_tuple_desc->slots().size(); ++i) { - int j = index_map_inv[i]; - auto slot_desc = _dest_tuple_desc->slots()[j]; - src_block.insert(ColumnWithTypeAndName(slot_desc->get_empty_mutable_column(), - slot_desc->get_data_type_ptr(), - slot_desc->col_name())); + for (int i = 0; i < columns_desc.size(); ++i) { + TypeDescriptor descriptor(columns_desc[i].type); + auto data_type = + vectorized::DataTypeFactory::instance().create_data_type(descriptor, true); + src_block.insert(ColumnWithTypeAndName(data_type->create_column(), data_type, + columns_desc[i].name)); } while (true) { RETURN_IF_CANCELLED(state); @@ -319,171 +279,6 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, return Status::OK(); } -Status VSchemaScanNode::write_slot_to_vectorized_column(void* slot, SlotDescriptor* slot_desc, - vectorized::MutableColumnPtr* column_ptr) { - vectorized::IColumn* col_ptr = column_ptr->get(); - if (slot_desc->is_nullable()) { - auto* nullable_column = reinterpret_cast(column_ptr->get()); - nullable_column->get_null_map_data().push_back(0); - col_ptr = &nullable_column->get_nested_column(); - } - switch (slot_desc->type().type) { - case TYPE_HLL: { - HyperLogLog* hll_slot = reinterpret_cast(slot); - reinterpret_cast(col_ptr)->get_data().emplace_back(*hll_slot); - break; - } - case TYPE_VARCHAR: - case TYPE_CHAR: - case TYPE_STRING: { - StringRef* str_slot = reinterpret_cast(slot); - reinterpret_cast(col_ptr)->insert_data(str_slot->data, - str_slot->size); - break; - } - - case TYPE_BOOLEAN: { - uint8_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_TINYINT: { - int8_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_SMALLINT: { - int16_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_INT: { - int32_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_BIGINT: { - int64_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_LARGEINT: { - __int128 num; - memcpy(&num, slot, sizeof(__int128)); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_FLOAT: { - float num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value( - num); - break; - } - - case TYPE_DOUBLE: { - double num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value( - num); - break; - } - - case TYPE_DATE: { - VecDateTimeValue value; - DateTimeValue* ts_slot = reinterpret_cast(slot); - value.convert_dt_to_vec_dt(ts_slot); - reinterpret_cast*>(col_ptr)->insert_data( - reinterpret_cast(&value), 0); - break; - } - - case TYPE_DATEV2: { - uint32_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_DATETIME: { - VecDateTimeValue value; - DateTimeValue* ts_slot = reinterpret_cast(slot); - value.convert_dt_to_vec_dt(ts_slot); - reinterpret_cast*>(col_ptr)->insert_data( - reinterpret_cast(&value), 0); - break; - } - - case TYPE_DATETIMEV2: { - uint32_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_DECIMALV2: { - const Int128 num = (reinterpret_cast(slot))->value; - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - case TYPE_DECIMAL128I: { - const Int128 num = (reinterpret_cast(slot))->value; - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - case TYPE_DECIMAL32: { - const int32_t num = *reinterpret_cast(slot); - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - case TYPE_DECIMAL64: { - const int64_t num = *reinterpret_cast(slot); - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - default: { - DCHECK(false) << "bad slot type: " << slot_desc->type(); - std::stringstream ss; - ss << "Fail to convert schema type:'" << slot_desc->type() << " on column:`" - << slot_desc->col_name() + "`"; - return Status::InternalError(ss.str()); - } - } - - return Status::OK(); -} - -void VSchemaScanNode::project_tuple() { - memset(_dest_single_tuple, 0, _dest_tuple_desc->num_null_bytes()); - - for (int i = 0; i < _slot_num; ++i) { - if (!_dest_tuple_desc->slots()[i]->is_materialized()) { - continue; - } - int j = _index_map[i]; - - if (_src_single_tuple->is_null(_src_tuple_desc->slots()[j]->null_indicator_offset())) { - _dest_single_tuple->set_null(_dest_tuple_desc->slots()[i]->null_indicator_offset()); - } else { - void* dest_slot = - _dest_single_tuple->get_slot(_dest_tuple_desc->slots()[i]->tuple_offset()); - void* src_slot = - _src_single_tuple->get_slot(_src_tuple_desc->slots()[j]->tuple_offset()); - int slot_size = _src_tuple_desc->slots()[j]->type().get_slot_size(); - memcpy(dest_slot, src_slot, slot_size); - } - } -} - Status VSchemaScanNode::close(RuntimeState* state) { if (is_closed()) { return Status::OK(); diff --git a/be/src/vec/exec/vschema_scan_node.h b/be/src/vec/exec/vschema_scan_node.h index 57117b23af6b79..92df6279a81b27 100644 --- a/be/src/vec/exec/vschema_scan_node.h +++ b/be/src/vec/exec/vschema_scan_node.h @@ -37,7 +37,7 @@ class VSchemaScanNode : public ScanNode { VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); ~VSchemaScanNode(); Status prepare(RuntimeState* state) override; - virtual Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos) override; + Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos) override; // Prepare conjuncts, create Schema columns to slots mapping // initialize schema_scanner @@ -60,8 +60,6 @@ class VSchemaScanNode : public ScanNode { // Tuple id resolved in prepare() to set _tuple_desc; TupleId _tuple_id; - // Descriptor of tuples read from schema table. - const TupleDescriptor* _src_tuple_desc; // Descriptor of dest tuples const TupleDescriptor* _dest_tuple_desc; // Tuple index in tuple row. @@ -72,16 +70,6 @@ class VSchemaScanNode : public ScanNode { std::unique_ptr _tuple_pool; // Jni helper for scanning an schema table. std::unique_ptr _schema_scanner; - // Current tuple. - doris::Tuple* _src_tuple; - // Map from index in slots to column of schema table. - std::vector _index_map; - - Status write_slot_to_vectorized_column(void* slot, SlotDescriptor* slot_desc, - vectorized::MutableColumnPtr* col_ptr); - void project_tuple(); - doris::Tuple* _src_single_tuple; - doris::Tuple* _dest_single_tuple; }; } // namespace vectorized } // namespace doris \ No newline at end of file From e1a11464f0dfc7de0ded6d525815f0c5e3ade727 Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Sun, 29 Jan 2023 18:05:23 +0800 Subject: [PATCH 23/24] fix --- be/src/vec/exec/vschema_scan_node.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index 7d02863f7b7cbd..4b6eda4ba95b1f 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -222,19 +222,18 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, RETURN_IF_CANCELLED(state); bool schema_eos = false; - block->clear(); const std::vector& columns_desc(_schema_scanner->get_column_desc()); - for (int i = 0; i < _slot_num; ++i) { - auto dest_slot_desc = _dest_tuple_desc->slots()[i]; - block->insert(ColumnWithTypeAndName(dest_slot_desc->get_empty_mutable_column(), - dest_slot_desc->get_data_type_ptr(), - dest_slot_desc->col_name())); - } - do { - vectorized::Block src_block; + block->clear(); + for (int i = 0; i < _slot_num; ++i) { + auto dest_slot_desc = _dest_tuple_desc->slots()[i]; + block->insert(ColumnWithTypeAndName(dest_slot_desc->get_empty_mutable_column(), + dest_slot_desc->get_data_type_ptr(), + dest_slot_desc->col_name())); + } + vectorized::Block src_block; for (int i = 0; i < columns_desc.size(); ++i) { TypeDescriptor descriptor(columns_desc[i].type); auto data_type = From 4f84de5d41ffc80db7d7efa69ce2888d926049c5 Mon Sep 17 00:00:00 2001 From: Cai-Yao <729673078@qq.com> Date: Mon, 30 Jan 2023 01:14:46 +0800 Subject: [PATCH 24/24] fix be ut bug --- be/src/exec/schema_scanner.cpp | 74 +++++++++++++++++++ be/src/exec/schema_scanner.h | 4 +- .../vec/exec/parquet/parquet_thrift_test.cpp | 10 +-- be/test/vec/exprs/vexpr_test.cpp | 5 +- 4 files changed, 84 insertions(+), 9 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 8e05320d89db0e..36c7459d9fd275 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -85,6 +85,8 @@ Status SchemaScanner::init(SchemaScannerParam* param, ObjectPool* pool) { return Status::InternalError("invalid parameter"); } + RETURN_IF_ERROR(create_tuple_desc(pool)); + _param = param; _is_init = true; @@ -282,4 +284,76 @@ Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, return Status::OK(); } +Status SchemaScanner::create_tuple_desc(ObjectPool* pool) { + int null_column = 0; + for (int i = 0; i < _columns.size(); ++i) { + if (_columns[i].is_null) { + null_column++; + } + } + + int offset = (null_column + 7) / 8; + std::vector slots; + int null_byte = 0; + int null_bit = 0; + + for (int i = 0; i < _columns.size(); ++i) { + TSlotDescriptor t_slot_desc; + if (_columns[i].type == TYPE_DECIMALV2) { + t_slot_desc.__set_slotType(TypeDescriptor::create_decimalv2_type(27, 9).to_thrift()); + } else { + TypeDescriptor descriptor(_columns[i].type); + if (_columns[i].precision >= 0 && _columns[i].scale >= 0) { + descriptor.precision = _columns[i].precision; + descriptor.scale = _columns[i].scale; + } + t_slot_desc.__set_slotType(descriptor.to_thrift()); + } + t_slot_desc.__set_colName(_columns[i].name); + t_slot_desc.__set_columnPos(i); + t_slot_desc.__set_byteOffset(offset); + + if (_columns[i].is_null) { + t_slot_desc.__set_nullIndicatorByte(null_byte); + t_slot_desc.__set_nullIndicatorBit(null_bit); + null_bit = (null_bit + 1) % 8; + + if (0 == null_bit) { + null_byte++; + } + } else { + t_slot_desc.__set_nullIndicatorByte(0); + t_slot_desc.__set_nullIndicatorBit(-1); + } + + t_slot_desc.id = i; + t_slot_desc.__set_slotIdx(i); + t_slot_desc.__set_isMaterialized(true); + + SlotDescriptor* slot = pool->add(new (std::nothrow) SlotDescriptor(t_slot_desc)); + + if (nullptr == slot) { + return Status::InternalError("no memory for _tuple_desc."); + } + + slots.push_back(slot); + offset += _columns[i].size; + } + + TTupleDescriptor t_tuple_desc; + t_tuple_desc.__set_byteSize(offset); + t_tuple_desc.__set_numNullBytes((null_byte * 8 + null_bit + 7) / 8); + _tuple_desc = pool->add(new (std::nothrow) TupleDescriptor(t_tuple_desc)); + + if (nullptr == _tuple_desc) { + return Status::InternalError("no memory for _tuple_desc."); + } + + for (int i = 0; i < slots.size(); ++i) { + _tuple_desc->add_slot(slots[i]); + } + + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index 8e794792973416..c9bae906e260eb 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -87,19 +87,21 @@ class SchemaScanner { const std::vector& get_column_desc() const { return _columns; } // factory function static SchemaScanner* create(TSchemaTableType::type type); - + const TupleDescriptor* tuple_desc() const { return _tuple_desc; } const TSchemaTableType::type type() const { return _schema_table_type; } static void set_doris_server(DorisServer* doris_server) { _s_doris_server = doris_server; } protected: Status fill_dest_column(vectorized::Block* block, void* data, const ColumnDesc& slot_desc); + Status create_tuple_desc(ObjectPool* pool); bool _is_init; // this is used for sub class SchemaScannerParam* _param; // schema table's column desc std::vector _columns; + TupleDescriptor* _tuple_desc; static DorisServer* _s_doris_server; diff --git a/be/test/vec/exec/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/parquet/parquet_thrift_test.cpp index c838d11b928555..b6d635aa44a0e3 100644 --- a/be/test/vec/exec/parquet/parquet_thrift_test.cpp +++ b/be/test/vec/exec/parquet/parquet_thrift_test.cpp @@ -227,7 +227,7 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column static void create_block(std::unique_ptr& block) { // Current supported column type: - SchemaScanner::ColumnDesc column_descs[] = { + std::vector column_descs = { {"tinyint_col", TYPE_TINYINT, sizeof(int8_t), true}, {"smallint_col", TYPE_SMALLINT, sizeof(int16_t), true}, {"int_col", TYPE_INT, sizeof(int32_t), true}, @@ -246,8 +246,7 @@ static void create_block(std::unique_ptr& block) { {"date_col", TYPE_DATE, sizeof(DateTimeValue), true}, {"date_v2_col", TYPE_DATEV2, sizeof(uint32_t), true}, {"timestamp_v2_col", TYPE_DATETIMEV2, sizeof(DateTimeValue), true, 18, 0}}; - SchemaScanner schema_scanner(column_descs, - sizeof(column_descs) / sizeof(SchemaScanner::ColumnDesc)); + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool); @@ -347,7 +346,7 @@ TEST_F(ParquetThriftReaderTest, dict_decoder) { } TEST_F(ParquetThriftReaderTest, group_reader) { - SchemaScanner::ColumnDesc column_descs[] = { + std::vector column_descs = { {"tinyint_col", TYPE_TINYINT, sizeof(int8_t), true}, {"smallint_col", TYPE_SMALLINT, sizeof(int16_t), true}, {"int_col", TYPE_INT, sizeof(int32_t), true}, @@ -362,8 +361,7 @@ TEST_F(ParquetThriftReaderTest, group_reader) { {"char_col", TYPE_CHAR, sizeof(StringRef), true}, {"varchar_col", TYPE_VARCHAR, sizeof(StringRef), true}, {"date_col", TYPE_DATE, sizeof(DateTimeValue), true}}; - int num_cols = sizeof(column_descs) / sizeof(SchemaScanner::ColumnDesc); - SchemaScanner schema_scanner(column_descs, num_cols); + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool); diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp index cacc350303406d..0f1026a51a32c7 100644 --- a/be/test/vec/exprs/vexpr_test.cpp +++ b/be/test/vec/exprs/vexpr_test.cpp @@ -63,8 +63,9 @@ TEST(TEST_VEXPR, ABSTEST) { TEST(TEST_VEXPR, ABSTEST2) { using namespace doris; - SchemaScanner::ColumnDesc column_descs[] = {{"k1", TYPE_INT, sizeof(int32_t), false}}; - SchemaScanner schema_scanner(column_descs, 1); + std::vector column_descs = { + {"k1", TYPE_INT, sizeof(int32_t), false}}; + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool);