diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 4f2d029df761a4..36c7459d9fd275 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -33,34 +33,24 @@ #include "exec/schema_scanner/schema_variables_scanner.h" #include "exec/schema_scanner/schema_views_scanner.h" #include "runtime/define_primitive_type.h" +#include "vec/columns/column.h" #include "vec/common/string_ref.h" +#include "vec/core/block.h" namespace doris { DorisServer* SchemaScanner::_s_doris_server; -SchemaScanner::SchemaScanner(ColumnDesc* columns, int column_num) +SchemaScanner::SchemaScanner(const std::vector& columns) : _is_init(false), _param(nullptr), _columns(columns), - _column_num(column_num), - _tuple_desc(nullptr), _schema_table_type(TSchemaTableType::SCH_INVALID) {} -SchemaScanner::SchemaScanner(ColumnDesc* columns, int column_num, TSchemaTableType::type type) - : _is_init(false), - _param(nullptr), - _columns(columns), - _column_num(column_num), - _tuple_desc(nullptr), - _schema_table_type(type) {} +SchemaScanner::SchemaScanner(const std::vector& columns, TSchemaTableType::type type) + : _is_init(false), _param(nullptr), _columns(columns), _schema_table_type(type) {} -SchemaScanner::~SchemaScanner() { - if (_is_create_columns == true && _columns != nullptr) { - delete[] _columns; - _columns = nullptr; - } -} +SchemaScanner::~SchemaScanner() {} Status SchemaScanner::start(RuntimeState* state) { if (!_is_init) { @@ -70,12 +60,12 @@ Status SchemaScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } @@ -91,15 +81,12 @@ Status SchemaScanner::init(SchemaScannerParam* param, ObjectPool* pool) { return Status::InternalError("invalid parameter"); } - if (_schema_table_type == TSchemaTableType::SCH_BACKENDS) { - RETURN_IF_ERROR(create_columns(param->table_structure, pool)); - } - - if (nullptr == _columns) { + if (_columns.empty()) { return Status::InternalError("invalid parameter"); } RETURN_IF_ERROR(create_tuple_desc(pool)); + _param = param; _is_init = true; @@ -145,23 +132,161 @@ SchemaScanner* SchemaScanner::create(TSchemaTableType::type type) { } } -Status SchemaScanner::create_columns(const std::vector* table_structure, - ObjectPool* pool) { - _column_num = table_structure->size(); - _columns = new ColumnDesc[_column_num]; - _is_create_columns = true; - for (size_t idx = 0; idx < table_structure->size(); ++idx) { - _columns[idx].name = table_structure->at(idx).column_name.c_str(); - _columns[idx].type = thrift_to_type(table_structure->at(idx).type); - _columns[idx].size = table_structure->at(idx).len; - _columns[idx].is_null = table_structure->at(idx).is_null; +Status SchemaScanner::fill_dest_column(vectorized::Block* block, void* data, + const ColumnDesc& col_desc) { + if (!block->has(col_desc.name)) { + return Status::OK(); + } + vectorized::MutableColumnPtr column_ptr = + std::move(*block->get_by_name(col_desc.name).column).assume_mutable(); + vectorized::IColumn* col_ptr = column_ptr.get(); + + if (data == nullptr) { + auto* nullable_column = reinterpret_cast(col_ptr); + nullable_column->insert_data(nullptr, 0); + return Status::OK(); + } + auto* nullable_column = reinterpret_cast(col_ptr); + nullable_column->get_null_map_data().push_back(0); + col_ptr = &nullable_column->get_nested_column(); + switch (col_desc.type) { + case TYPE_HLL: { + HyperLogLog* hll_slot = reinterpret_cast(data); + reinterpret_cast(col_ptr)->get_data().emplace_back(*hll_slot); + break; + } + case TYPE_VARCHAR: + case TYPE_CHAR: + case TYPE_STRING: { + StringRef* str_slot = reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data(str_slot->data, + str_slot->size); + break; + } + + case TYPE_BOOLEAN: { + uint8_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_TINYINT: { + int8_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_SMALLINT: { + int16_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_INT: { + int32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_BIGINT: { + int64_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_LARGEINT: { + __int128 num; + memcpy(&num, data, sizeof(__int128)); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; } + + case TYPE_FLOAT: { + float num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_DOUBLE: { + double num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_DATE: { + vectorized::VecDateTimeValue value; + DateTimeValue* ts_slot = reinterpret_cast(data); + value.convert_dt_to_vec_dt(ts_slot); + reinterpret_cast*>(col_ptr)->insert_data( + reinterpret_cast(&value), 0); + break; + } + + case TYPE_DATEV2: { + uint32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_DATETIME: { + vectorized::VecDateTimeValue value; + DateTimeValue* ts_slot = reinterpret_cast(data); + value.convert_dt_to_vec_dt(ts_slot); + reinterpret_cast*>(col_ptr)->insert_data( + reinterpret_cast(&value), 0); + break; + } + + case TYPE_DATETIMEV2: { + uint32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value(num); + break; + } + + case TYPE_DECIMALV2: { + const vectorized::Int128 num = (reinterpret_cast(data))->value; + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + case TYPE_DECIMAL128I: { + const vectorized::Int128 num = (reinterpret_cast(data))->value; + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + case TYPE_DECIMAL32: { + const int32_t num = *reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + case TYPE_DECIMAL64: { + const int64_t num = *reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + default: { + DCHECK(false) << "bad slot type: " << col_desc.type; + std::stringstream ss; + ss << "Fail to convert schema type:'" << col_desc.type << " on column:`" + << std::string(col_desc.name) + "`"; + return Status::InternalError(ss.str()); + } + } + return Status::OK(); } Status SchemaScanner::create_tuple_desc(ObjectPool* pool) { int null_column = 0; - for (int i = 0; i < _column_num; ++i) { + for (int i = 0; i < _columns.size(); ++i) { if (_columns[i].is_null) { null_column++; } @@ -172,7 +297,7 @@ Status SchemaScanner::create_tuple_desc(ObjectPool* pool) { int null_byte = 0; int null_bit = 0; - for (int i = 0; i < _column_num; ++i) { + for (int i = 0; i < _columns.size(); ++i) { TSlotDescriptor t_slot_desc; if (_columns[i].type == TYPE_DECIMALV2) { t_slot_desc.__set_slotType(TypeDescriptor::create_decimalv2_type(27, 9).to_thrift()); diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index 9d2fa1c5fed672..c9bae906e260eb 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -25,6 +25,7 @@ #include "gen_cpp/Types_types.h" #include "runtime/mem_pool.h" #include "runtime/tuple.h" +#include "vec/core/block.h" namespace doris { @@ -32,6 +33,10 @@ namespace doris { class DorisServer; class RuntimeState; +namespace vectorized { +class Block; +} + // scanner parameter from frontend struct SchemaScannerParam { const std::string* db; @@ -70,40 +75,34 @@ class SchemaScanner { int precision = -1; int scale = -1; }; - SchemaScanner(ColumnDesc* columns, int column_num); - SchemaScanner(ColumnDesc* columns, int column_num, TSchemaTableType::type type); + SchemaScanner(const std::vector& columns); + SchemaScanner(const std::vector& columns, TSchemaTableType::type type); virtual ~SchemaScanner(); // init object need information, schema etc. virtual Status init(SchemaScannerParam* param, ObjectPool* pool); // Start to work virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + virtual Status get_next_block(vectorized::Block* block, bool* eos); + const std::vector& get_column_desc() const { return _columns; } // factory function static SchemaScanner* create(TSchemaTableType::type type); - const TupleDescriptor* tuple_desc() const { return _tuple_desc; } + const TSchemaTableType::type type() const { return _schema_table_type; } static void set_doris_server(DorisServer* doris_server) { _s_doris_server = doris_server; } protected: + Status fill_dest_column(vectorized::Block* block, void* data, const ColumnDesc& slot_desc); Status create_tuple_desc(ObjectPool* pool); - Status create_columns(const std::vector* table_structure, - ObjectPool* pool); bool _is_init; // this is used for sub class SchemaScannerParam* _param; - // pointer to schema table's column desc - ColumnDesc* _columns; - // num of columns - int _column_num; + // schema table's column desc + std::vector _columns; TupleDescriptor* _tuple_desc; - // _is_create_columns means if ColumnDesc is created from FE. - // `_columns` should be deleted if _is_create_columns = true. - bool _is_create_columns = false; - static DorisServer* _s_doris_server; TSchemaTableType::type _schema_table_type; diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.cpp b/be/src/exec/schema_scanner/schema_backends_scanner.cpp index 9c1b0c015a25f9..e9c008f8a23290 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_backends_scanner.cpp @@ -21,6 +21,7 @@ #include #include +#include "common/status.h" #include "exec/schema_scanner.h" #include "gen_cpp/FrontendService.h" #include "runtime/client_cache.h" @@ -32,8 +33,35 @@ namespace doris { +std::vector SchemaBackendsScanner::_s_tbls_columns = { + // name, type, size + {"BackendId", TYPE_BIGINT, sizeof(StringRef), false}, + {"TabletNum", TYPE_BIGINT, sizeof(StringRef), false}, + {"HeartbeatPort", TYPE_INT, sizeof(int), false}, + {"BePort", TYPE_INT, sizeof(int), false}, + {"HttpPort", TYPE_INT, sizeof(int), false}, + {"BrpcPort", TYPE_INT, sizeof(int), false}, + {"Cluster", TYPE_VARCHAR, sizeof(StringRef), false}, + {"IP", TYPE_VARCHAR, sizeof(StringRef), false}, + {"LastStartTime", TYPE_VARCHAR, sizeof(StringRef), false}, + {"LastHeartbeat", TYPE_VARCHAR, sizeof(StringRef), false}, + {"Alive", TYPE_VARCHAR, sizeof(StringRef), false}, + {"SystemDecommissioned", TYPE_VARCHAR, sizeof(StringRef), false}, + {"ClusterDecommissioned", TYPE_VARCHAR, sizeof(StringRef), false}, + {"DataUsedCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"AvailCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"TotalCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"UsedPct", TYPE_DOUBLE, sizeof(double), false}, + {"MaxDiskUsedPct", TYPE_DOUBLE, sizeof(double), false}, + {"RemoteUsedCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"Tag", TYPE_VARCHAR, sizeof(StringRef), false}, + {"ErrMsg", TYPE_VARCHAR, sizeof(StringRef), false}, + {"Version", TYPE_VARCHAR, sizeof(StringRef), false}, + {"Status", TYPE_VARCHAR, sizeof(StringRef), false}, +}; + SchemaBackendsScanner::SchemaBackendsScanner() - : SchemaScanner(nullptr, 0, TSchemaTableType::SCH_BACKENDS), _row_idx(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_BACKENDS) {} Status SchemaBackendsScanner::start(RuntimeState* state) { if (!_is_init) { @@ -44,61 +72,54 @@ Status SchemaBackendsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaBackendsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaBackendsScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - if (_row_idx >= _batch_data.size()) { - *eos = true; - return Status::OK(); - } - *eos = false; - return _fill_one_row(tuple, pool); + *eos = true; + return _fill_block_impl(block); } -Status SchemaBackendsScanner::_fill_one_row(Tuple* tuple, MemPool* pool) { - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - for (size_t col_idx = 0; col_idx < _column_num; ++col_idx) { - RETURN_IF_ERROR(_fill_one_col(tuple, pool, col_idx)); - } - ++_row_idx; - return Status::OK(); -} - -Status SchemaBackendsScanner::_fill_one_col(Tuple* tuple, MemPool* pool, size_t col_idx) { - auto it = _col_name_to_type.find(_columns[col_idx].name); - - // if this column is not exist in BE, we fill it with `NULL`. - if (it == _col_name_to_type.end()) { - if (_columns[col_idx].is_null) { - tuple->set_null(_tuple_desc->slots()[col_idx]->null_indicator_offset()); +Status SchemaBackendsScanner::_fill_block_impl(vectorized::Block* block) { + auto row_num = _batch_data.size(); + for (size_t col_idx = 0; col_idx < _columns.size(); ++col_idx) { + auto it = _col_name_to_type.find(_columns[col_idx].name); + if (it == _col_name_to_type.end()) { + if (_columns[col_idx].is_null) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + fill_dest_column(block, nullptr, _s_tbls_columns[col_idx]); + } + } else { + return Status::InternalError( + "column {} is not found in BE, and {} is not nullable.", + _columns[col_idx].name, _columns[col_idx].name); + } + } else if (it->second == TYPE_BIGINT) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].longVal, + _s_tbls_columns[col_idx]); + } + } else if (it->second == TYPE_INT) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].intVal, + _s_tbls_columns[col_idx]); + } + } else if (it->second == TYPE_VARCHAR) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].stringVal, + _s_tbls_columns[col_idx]); + } + } else if (it->second == TYPE_DOUBLE) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + fill_dest_column(block, &_batch_data[row_idx].column_value[col_idx].doubleVal, + _s_tbls_columns[col_idx]); + } } else { - return Status::InternalError("column {} is not found in BE, and {} is not nullable.", - _columns[col_idx].name, _columns[col_idx].name); + // other type } - } else if (it->second == TYPE_BIGINT) { - void* slot = tuple->get_slot(_tuple_desc->slots()[col_idx]->tuple_offset()); - *(reinterpret_cast(slot)) = _batch_data[_row_idx].column_value[col_idx].longVal; - } else if (it->second == TYPE_INT) { - void* slot = tuple->get_slot(_tuple_desc->slots()[col_idx]->tuple_offset()); - *(reinterpret_cast(slot)) = _batch_data[_row_idx].column_value[col_idx].intVal; - } else if (it->second == TYPE_VARCHAR) { - void* slot = tuple->get_slot(_tuple_desc->slots()[col_idx]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->data = - (char*)pool->allocate(_batch_data[_row_idx].column_value[col_idx].stringVal.size()); - str_slot->size = _batch_data[_row_idx].column_value[col_idx].stringVal.size(); - memcpy(const_cast(str_slot->data), - _batch_data[_row_idx].column_value[col_idx].stringVal.c_str(), str_slot->size); - } else if (it->second == TYPE_DOUBLE) { - void* slot = tuple->get_slot(_tuple_desc->slots()[col_idx]->tuple_offset()); - *(reinterpret_cast(slot)) = - _batch_data[_row_idx].column_value[col_idx].doubleVal; - } else { - // other type } return Status::OK(); } diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.h b/be/src/exec/schema_scanner/schema_backends_scanner.h index 32753f568e5b18..82ca121561c5f8 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.h +++ b/be/src/exec/schema_scanner/schema_backends_scanner.h @@ -17,6 +17,7 @@ #pragma once +#include "common/status.h" #include "exec/schema_scanner.h" namespace doris { @@ -26,19 +27,17 @@ class SchemaBackendsScanner : public SchemaScanner { ~SchemaBackendsScanner() override = default; Status start(RuntimeState* state) override; - Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status _fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); Status _fetch_backends_info(); - Status _fill_one_col(Tuple* tuple, MemPool* pool, size_t idx); Status _set_col_name_to_type(); -private: // column_name -> type, set by _set_col_name_to_type() std::unordered_map _col_name_to_type; + static std::vector _s_tbls_columns; std::vector _batch_data; - size_t _row_idx; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp index 0a96db94c4347d..56b446fa56878a 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp @@ -17,11 +17,12 @@ #include "exec/schema_scanner/schema_charsets_scanner.h" +#include "common/status.h" #include "vec/common/string_ref.h" namespace doris { -SchemaScanner::ColumnDesc SchemaCharsetsScanner::_s_css_columns[] = { +std::vector SchemaCharsetsScanner::_s_css_columns = { // name, type, size {"CHARACTER_SET_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, {"DEFAULT_COLLATE_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -35,70 +36,49 @@ SchemaCharsetsScanner::CharsetStruct SchemaCharsetsScanner::_s_charsets[] = { }; SchemaCharsetsScanner::SchemaCharsetsScanner() - : SchemaScanner(_s_css_columns, sizeof(_s_css_columns) / sizeof(SchemaScanner::ColumnDesc)), - _index(0) {} + : SchemaScanner(_s_css_columns, TSchemaTableType::SCH_CHARSETS) {} SchemaCharsetsScanner::~SchemaCharsetsScanner() {} -Status SchemaCharsetsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { +Status SchemaCharsetsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("call this before initial."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("invalid parameter."); + } + *eos = true; + return _fill_block_impl(block); +} + +Status SchemaCharsetsScanner::_fill_block_impl(vectorized::Block* block) { + auto row_num = 0; + while (nullptr != _s_charsets[row_num].charset) { + ++row_num; + } + // variables names - { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_charsets[_index].charset); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_charsets[_index].charset, len + 1); - str_slot->size = len; + for (int i = 0; i < row_num; ++i) { + StringRef str = StringRef(_s_charsets[i].charset, strlen(_s_charsets[i].charset)); + fill_dest_column(block, &str, _s_css_columns[0]); } // DEFAULT_COLLATE_NAME - { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_charsets[_index].default_collation); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_charsets[_index].default_collation, len + 1); - str_slot->size = len; + for (int i = 0; i < row_num; ++i) { + StringRef str = StringRef(_s_charsets[i].default_collation, + strlen(_s_charsets[i].default_collation)); + fill_dest_column(block, &str, _s_css_columns[1]); } // DESCRIPTION - { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_charsets[_index].description); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_charsets[_index].description, len + 1); - str_slot->size = len; + for (int i = 0; i < row_num; ++i) { + StringRef str = StringRef(_s_charsets[i].description, strlen(_s_charsets[i].description)); + fill_dest_column(block, &str, _s_css_columns[2]); } // maxlen - { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - *(int64_t*)slot = _s_charsets[_index].maxlen; + for (int i = 0; i < row_num; ++i) { + int64_t src = _s_charsets[i].maxlen; + fill_dest_column(block, &src, _s_css_columns[3]); } - _index++; return Status::OK(); } -Status SchemaCharsetsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("call this before initial."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("invalid parameter."); - } - if (nullptr == _s_charsets[_index].charset) { - *eos = true; - return Status::OK(); - } - *eos = false; - return fill_one_row(tuple, pool); -} - } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.h b/be/src/exec/schema_scanner/schema_charsets_scanner.h index 3a9f07100ae342..37b2866bab31e3 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.h @@ -26,9 +26,9 @@ namespace doris { class SchemaCharsetsScanner : public SchemaScanner { public: SchemaCharsetsScanner(); - virtual ~SchemaCharsetsScanner(); + ~SchemaCharsetsScanner() override; - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: struct CharsetStruct { @@ -38,10 +38,9 @@ class SchemaCharsetsScanner : public SchemaScanner { int64_t maxlen; }; - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); - int _index; - static SchemaScanner::ColumnDesc _s_css_columns[]; + static std::vector _s_css_columns; static CharsetStruct _s_charsets[]; }; diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.cpp b/be/src/exec/schema_scanner/schema_collations_scanner.cpp index 3201deb68bfcc6..eb35c30f2c5a4e 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_collations_scanner.cpp @@ -17,12 +17,13 @@ #include "exec/schema_scanner/schema_collations_scanner.h" +#include "common/status.h" #include "runtime/primitive_type.h" #include "vec/common/string_ref.h" namespace doris { -SchemaScanner::ColumnDesc SchemaCollationsScanner::_s_cols_columns[] = { +std::vector SchemaCollationsScanner::_s_cols_columns = { // name, type, size {"COLLATION_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, {"CHARACTER_SET_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -38,89 +39,58 @@ SchemaCollationsScanner::CollationStruct SchemaCollationsScanner::_s_collations[ }; SchemaCollationsScanner::SchemaCollationsScanner() - : SchemaScanner(_s_cols_columns, - sizeof(_s_cols_columns) / sizeof(SchemaScanner::ColumnDesc)), - _index(0) {} + : SchemaScanner(_s_cols_columns, TSchemaTableType::SCH_COLLATIONS) {} SchemaCollationsScanner::~SchemaCollationsScanner() {} -Status SchemaCollationsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { +Status SchemaCollationsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("call this before initial."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("invalid parameter."); + } + + *eos = true; + return _fill_block_impl(block); +} + +Status SchemaCollationsScanner::_fill_block_impl(vectorized::Block* block) { + auto row_num = 0; + while (nullptr != _s_collations[row_num].name) { + ++row_num; + } // COLLATION_NAME - { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_collations[_index].name); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_collations[_index].name, len + 1); - str_slot->size = len; + for (int i = 0; i < row_num; ++i) { + StringRef str = StringRef(_s_collations[i].name, strlen(_s_collations[i].name)); + fill_dest_column(block, &str, _s_cols_columns[0]); } // charset - { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_collations[_index].charset); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_collations[_index].charset, len + 1); - str_slot->size = len; + for (int i = 0; i < row_num; ++i) { + StringRef str = StringRef(_s_collations[i].charset, strlen(_s_collations[i].charset)); + fill_dest_column(block, &str, _s_cols_columns[1]); } // id - { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - *(int64_t*)slot = _s_collations[_index].id; + for (int i = 0; i < row_num; ++i) { + int64_t src = _s_collations[i].id; + fill_dest_column(block, &src, _s_cols_columns[2]); } // is_default - { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_collations[_index].is_default); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_collations[_index].is_default, len + 1); - str_slot->size = len; + for (int i = 0; i < row_num; ++i) { + StringRef str = StringRef(_s_collations[i].is_default, strlen(_s_collations[i].is_default)); + fill_dest_column(block, &str, _s_cols_columns[3]); } // IS_COMPILED - { - void* slot = tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_s_collations[_index].is_compile); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); - } - memcpy(const_cast(str_slot->data), _s_collations[_index].is_compile, len + 1); - str_slot->size = len; + for (int i = 0; i < row_num; ++i) { + StringRef str = StringRef(_s_collations[i].is_compile, strlen(_s_collations[i].is_compile)); + fill_dest_column(block, &str, _s_cols_columns[4]); } // sortlen - { - void* slot = tuple->get_slot(_tuple_desc->slots()[5]->tuple_offset()); - *(int64_t*)slot = _s_collations[_index].sortlen; + for (int i = 0; i < row_num; ++i) { + int64_t src = _s_collations[i].sortlen; + fill_dest_column(block, &src, _s_cols_columns[5]); } - _index++; return Status::OK(); } -Status SchemaCollationsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("call this before initial."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("invalid parameter."); - } - if (nullptr == _s_collations[_index].name) { - *eos = true; - return Status::OK(); - } - - *eos = false; - return fill_one_row(tuple, pool); -} - } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.h b/be/src/exec/schema_scanner/schema_collations_scanner.h index 66e3faeed33e69..d421cf3818365f 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.h +++ b/be/src/exec/schema_scanner/schema_collations_scanner.h @@ -26,9 +26,9 @@ namespace doris { class SchemaCollationsScanner : public SchemaScanner { public: SchemaCollationsScanner(); - virtual ~SchemaCollationsScanner(); + ~SchemaCollationsScanner() override; - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_block(vectorized::Block* block, bool* eos) override; private: struct CollationStruct { @@ -40,10 +40,9 @@ class SchemaCollationsScanner : public SchemaScanner { int64_t sortlen; }; - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); - int _index; - static SchemaScanner::ColumnDesc _s_cols_columns[]; + static std::vector _s_cols_columns; static CollationStruct _s_collations[]; }; diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp index 60a133b6d0c00f..8344a67484a847 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp @@ -17,6 +17,8 @@ #include "exec/schema_scanner/schema_columns_scanner.h" +#include +#include #include #include "exec/schema_scanner/schema_helper.h" @@ -25,7 +27,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaColumnsScanner::_s_col_columns[] = { +std::vector SchemaColumnsScanner::_s_col_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -54,10 +56,9 @@ SchemaScanner::ColumnDesc SchemaColumnsScanner::_s_col_columns[] = { }; SchemaColumnsScanner::SchemaColumnsScanner() - : SchemaScanner(_s_col_columns, sizeof(_s_col_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_col_columns, TSchemaTableType::SCH_COLUMNS), _db_index(0), - _table_index(0), - _column_index(0) {} + _table_index(0) {} SchemaColumnsScanner::~SchemaColumnsScanner() = default; @@ -95,7 +96,7 @@ Status SchemaColumnsScanner::start(RuntimeState* state) { } //For compatibility with mysql the result of DATA_TYPE in information_schema.columns -std::string SchemaColumnsScanner::to_mysql_data_type_string(TColumnDesc& desc) { +std::string SchemaColumnsScanner::_to_mysql_data_type_string(TColumnDesc& desc) { switch (desc.columnType) { case TPrimitiveType::BOOLEAN: return "tinyint"; @@ -136,7 +137,7 @@ std::string SchemaColumnsScanner::to_mysql_data_type_string(TColumnDesc& desc) { } } -std::string SchemaColumnsScanner::type_to_string(TColumnDesc& desc) { +std::string SchemaColumnsScanner::_type_to_string(TColumnDesc& desc) { switch (desc.columnType) { case TPrimitiveType::BOOLEAN: return "tinyint(1)"; @@ -218,320 +219,320 @@ std::string SchemaColumnsScanner::type_to_string(TColumnDesc& desc) { } } -// TODO: ALL schema algorithm like these need to be refactor to avoid UB and -// keep StringRef semantic to be kept. +Status SchemaColumnsScanner::_get_new_desc() { + TDescribeTableParams desc_params; + desc_params.__set_db(_db_result.dbs[_db_index - 1]); + if (_db_result.__isset.catalogs) { + desc_params.__set_catalog(_db_result.catalogs[_db_index - 1]); + } + desc_params.__set_table_name(_table_result.tables[_table_index++]); + if (nullptr != _param->current_user_ident) { + desc_params.__set_current_user_ident(*(_param->current_user_ident)); + } else { + if (nullptr != _param->user) { + desc_params.__set_user(*(_param->user)); + } + if (nullptr != _param->user_ip) { + desc_params.__set_user_ip(*(_param->user_ip)); + } + } + + if (nullptr != _param->ip && 0 != _param->port) { + RETURN_IF_ERROR(SchemaHelper::describe_table(*(_param->ip), _param->port, desc_params, + &_desc_result)); + } else { + return Status::InternalError("IP or port doesn't exists"); + } -//fill row in the "INFORMATION_SCHEMA COLUMNS" -//Reference from https://dev.mysql.com/doc/refman/8.0/en/information-schema-columns-table.html -Status SchemaColumnsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); + return Status::OK(); +} + +Status SchemaColumnsScanner::_get_new_table() { + TGetTablesParams table_params; + table_params.__set_db(_db_result.dbs[_db_index]); + if (_db_result.__isset.catalogs) { + table_params.__set_catalog(_db_result.catalogs[_db_index]); + } + _db_index++; + if (nullptr != _param->table) { + table_params.__set_pattern(*(_param->table)); + } + if (nullptr != _param->current_user_ident) { + table_params.__set_current_user_ident(*(_param->current_user_ident)); + } else { + if (nullptr != _param->user) { + table_params.__set_user(*(_param->user)); + } + if (nullptr != _param->user_ip) { + table_params.__set_user_ip(*(_param->user_ip)); + } + } + + if (nullptr != _param->ip && 0 != _param->port) { + RETURN_IF_ERROR(SchemaHelper::get_table_names(*(_param->ip), _param->port, table_params, + &_table_result)); + } else { + return Status::InternalError("IP or port doesn't exists"); + } + _table_index = 0; + return Status::OK(); +} + +Status SchemaColumnsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("use this class before inited."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input parameter is nullptr."); + } + + while (_table_index >= _table_result.tables.size()) { + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(_get_new_table()); + } else { + *eos = true; + return Status::OK(); + } + } + RETURN_IF_ERROR(_get_new_desc()); + + *eos = false; + return _fill_block_impl(block); +} + +Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { + auto columns_num = _desc_result.columns.size(); // TABLE_CATALOG { if (!_db_result.__isset.catalogs) { - tuple->set_null(_tuple_desc->slots()[0]->null_indicator_offset()); + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _s_col_columns[0]); + } } else { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - // todo: we may change all of StringRef in similar usage - // to Slice someday to distinguish different purposes of use. - // or just merge them. std::string catalog_name = _db_result.catalogs[_db_index - 1]; - str_slot->data = (char*)pool->allocate(catalog_name.size()); - str_slot->size = catalog_name.size(); - memcpy(const_cast(str_slot->data), catalog_name.c_str(), str_slot->size); + StringRef str = StringRef(catalog_name.c_str(), catalog_name.size()); + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, &str, _s_col_columns[0]); + } } } // TABLE_SCHEMA { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); - str_slot->data = (char*)pool->allocate(db_name.size()); - str_slot->size = db_name.size(); - memcpy(const_cast(str_slot->data), db_name.c_str(), str_slot->size); + StringRef str = StringRef(db_name.c_str(), db_name.size()); + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, &str, _s_col_columns[1]); + } } // TABLE_NAME { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->data = (char*)pool->allocate(_table_result.tables[_table_index - 1].length()); - str_slot->size = _table_result.tables[_table_index - 1].length(); - memcpy(const_cast(str_slot->data), _table_result.tables[_table_index - 1].c_str(), - str_slot->size); + StringRef str = StringRef(_table_result.tables[_table_index - 1].c_str(), + _table_result.tables[_table_index - 1].length()); + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, &str, _s_col_columns[2]); + } } // COLUMN_NAME { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->data = (char*)pool->allocate( - _desc_result.columns[_column_index].columnDesc.columnName.length()); - str_slot->size = _desc_result.columns[_column_index].columnDesc.columnName.length(); - memcpy(const_cast(str_slot->data), - _desc_result.columns[_column_index].columnDesc.columnName.c_str(), str_slot->size); + for (int i = 0; i < columns_num; ++i) { + StringRef str = StringRef(_desc_result.columns[i].columnDesc.columnName.c_str(), + _desc_result.columns[i].columnDesc.columnName.length()); + fill_dest_column(block, &str, _s_col_columns[3]); + } } // ORDINAL_POSITION { - void* slot = tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset()); - int64_t* bigint_slot = reinterpret_cast(slot); - *bigint_slot = _column_index + 1; + for (int i = 0; i < columns_num; ++i) { + int64_t src = i + 1; + fill_dest_column(block, &src, _s_col_columns[4]); + } } // COLUMN_DEFAULT - { tuple->set_null(_tuple_desc->slots()[5]->null_indicator_offset()); } + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _s_col_columns[5]); + } + } // IS_NULLABLE { - void* slot = tuple->get_slot(_tuple_desc->slots()[6]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - - if (_desc_result.columns[_column_index].columnDesc.__isset.isAllowNull) { - if (_desc_result.columns[_column_index].columnDesc.isAllowNull) { - str_slot->size = strlen("YES"); - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "YES", str_slot->size); + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.isAllowNull) { + if (_desc_result.columns[i].columnDesc.isAllowNull) { + StringRef str = StringRef("YES", 3); + fill_dest_column(block, &str, _s_col_columns[6]); + } else { + StringRef str = StringRef("NO", 2); + fill_dest_column(block, &str, _s_col_columns[6]); + } } else { - str_slot->size = strlen("NO"); - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "NO", str_slot->size); + StringRef str = StringRef("NO", 2); + fill_dest_column(block, &str, _s_col_columns[6]); } - } else { - str_slot->size = strlen("NO"); - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "NO", str_slot->size); } } // DATA_TYPE { - void* slot = tuple->get_slot(_tuple_desc->slots()[7]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string buffer = - to_mysql_data_type_string(_desc_result.columns[_column_index].columnDesc); - str_slot->size = buffer.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), buffer.c_str(), str_slot->size); + for (int i = 0; i < columns_num; ++i) { + std::string buffer = _to_mysql_data_type_string(_desc_result.columns[i].columnDesc); + StringRef str = StringRef(buffer.c_str(), buffer.length()); + fill_dest_column(block, &str, _s_col_columns[7]); + } } // CHARACTER_MAXIMUM_LENGTH // For string columns, the maximum length in characters. { - int data_type = _desc_result.columns[_column_index].columnDesc.columnType; - if (data_type == TPrimitiveType::VARCHAR || data_type == TPrimitiveType::CHAR || - data_type == TPrimitiveType::STRING) { - void* slot = tuple->get_slot(_tuple_desc->slots()[8]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnLength) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnLength; + for (int i = 0; i < columns_num; ++i) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (data_type == TPrimitiveType::VARCHAR || data_type == TPrimitiveType::CHAR || + data_type == TPrimitiveType::STRING) { + if (_desc_result.columns[i].columnDesc.__isset.columnLength) { + int64_t src = _desc_result.columns[i].columnDesc.columnLength; + fill_dest_column(block, &src, _s_col_columns[8]); + } else { + fill_dest_column(block, nullptr, _s_col_columns[8]); + } } else { - tuple->set_null(_tuple_desc->slots()[8]->null_indicator_offset()); + fill_dest_column(block, nullptr, _s_col_columns[8]); } - } else { - tuple->set_null(_tuple_desc->slots()[8]->null_indicator_offset()); } } // CHARACTER_OCTET_LENGTH // For string columns, the maximum length in bytes. { - int data_type = _desc_result.columns[_column_index].columnDesc.columnType; - if (data_type == TPrimitiveType::VARCHAR || data_type == TPrimitiveType::CHAR || - data_type == TPrimitiveType::STRING) { - void* slot = tuple->get_slot(_tuple_desc->slots()[9]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnLength) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnLength * 4; + for (int i = 0; i < columns_num; ++i) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (data_type == TPrimitiveType::VARCHAR || data_type == TPrimitiveType::CHAR || + data_type == TPrimitiveType::STRING) { + if (_desc_result.columns[i].columnDesc.__isset.columnLength) { + int64_t src = _desc_result.columns[i].columnDesc.columnLength * 4; + fill_dest_column(block, &src, _s_col_columns[9]); + } else { + fill_dest_column(block, nullptr, _s_col_columns[9]); + } } else { - tuple->set_null(_tuple_desc->slots()[9]->null_indicator_offset()); + fill_dest_column(block, nullptr, _s_col_columns[9]); } - } else { - tuple->set_null(_tuple_desc->slots()[9]->null_indicator_offset()); } } // NUMERIC_PRECISION { - void* slot = tuple->get_slot(_tuple_desc->slots()[10]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnPrecision) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnPrecision; - } else { - tuple->set_null(_tuple_desc->slots()[10]->null_indicator_offset()); + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.columnPrecision) { + int64_t src = _desc_result.columns[i].columnDesc.columnPrecision; + fill_dest_column(block, &src, _s_col_columns[10]); + } else { + fill_dest_column(block, nullptr, _s_col_columns[10]); + } } } // NUMERIC_SCALE { - void* slot = tuple->get_slot(_tuple_desc->slots()[11]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnScale) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnScale; - } else { - tuple->set_null(_tuple_desc->slots()[11]->null_indicator_offset()); + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.columnScale) { + int64_t src = _desc_result.columns[i].columnDesc.columnScale; + fill_dest_column(block, &src, _s_col_columns[11]); + } else { + fill_dest_column(block, nullptr, _s_col_columns[11]); + } } } // DATETIME_PRECISION - { tuple->set_null(_tuple_desc->slots()[12]->null_indicator_offset()); } + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _s_col_columns[12]); + } + } // CHARACTER_SET_NAME - { tuple->set_null(_tuple_desc->slots()[13]->null_indicator_offset()); } + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _s_col_columns[13]); + } + } // COLLATION_NAME - { tuple->set_null(_tuple_desc->slots()[14]->null_indicator_offset()); } + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _s_col_columns[14]); + } + } // COLUMN_TYPE { - void* slot = tuple->get_slot(_tuple_desc->slots()[15]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string buffer = type_to_string(_desc_result.columns[_column_index].columnDesc); - str_slot->size = buffer.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), buffer.c_str(), str_slot->size); + for (int i = 0; i < columns_num; ++i) { + std::string buffer = _type_to_string(_desc_result.columns[i].columnDesc); + StringRef str = StringRef(buffer.c_str(), buffer.length()); + fill_dest_column(block, &str, _s_col_columns[15]); + } } // COLUMN_KEY { - void* slot = tuple->get_slot(_tuple_desc->slots()[16]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnKey) { - str_slot->size = _desc_result.columns[_column_index].columnDesc.columnKey.length(); - str_slot->data = (char*)pool->allocate( - _desc_result.columns[_column_index].columnDesc.columnKey.length()); - memcpy(const_cast(str_slot->data), - _desc_result.columns[_column_index].columnDesc.columnKey.c_str(), - str_slot->size); - } else { - str_slot->size = strlen("") + 1; - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "", str_slot->size); + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.columnKey) { + StringRef str = StringRef(_desc_result.columns[i].columnDesc.columnKey.c_str(), + _desc_result.columns[i].columnDesc.columnKey.length()); + fill_dest_column(block, &str, _s_col_columns[16]); + } else { + StringRef str = StringRef("", 0); + fill_dest_column(block, &str, _s_col_columns[16]); + } } } // EXTRA { - void* slot = tuple->get_slot(_tuple_desc->slots()[17]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = strlen("") + 1; - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "", str_slot->size); + for (int i = 0; i < columns_num; ++i) { + StringRef str = StringRef("", 0); + fill_dest_column(block, &str, _s_col_columns[17]); + } } // PRIVILEGES { - void* slot = tuple->get_slot(_tuple_desc->slots()[18]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = strlen("") + 1; - str_slot->data = (char*)pool->allocate(str_slot->size); - memcpy(const_cast(str_slot->data), "", str_slot->size); + for (int i = 0; i < columns_num; ++i) { + StringRef str = StringRef("", 0); + fill_dest_column(block, &str, _s_col_columns[18]); + } } // COLUMN_COMMENT { - void* slot = tuple->get_slot(_tuple_desc->slots()[19]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->data = - (char*)pool->allocate(_desc_result.columns[_column_index].comment.length()); - str_slot->size = _desc_result.columns[_column_index].comment.length(); - memcpy(const_cast(str_slot->data), - _desc_result.columns[_column_index].comment.c_str(), str_slot->size); + for (int i = 0; i < columns_num; ++i) { + StringRef str = StringRef(_desc_result.columns[i].comment.c_str(), + _desc_result.columns[i].comment.length()); + fill_dest_column(block, &str, _s_col_columns[19]); + } } // COLUMN_SIZE { - void* slot = tuple->get_slot(_tuple_desc->slots()[20]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnLength) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnLength; - } else { - tuple->set_null(_tuple_desc->slots()[20]->null_indicator_offset()); + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.columnLength) { + int64_t src = _desc_result.columns[i].columnDesc.columnLength; + fill_dest_column(block, &src, _s_col_columns[20]); + } else { + fill_dest_column(block, nullptr, _s_col_columns[20]); + } } } // DECIMAL_DIGITS { - void* slot = tuple->get_slot(_tuple_desc->slots()[21]->tuple_offset()); - int64_t* str_slot = reinterpret_cast(slot); - if (_desc_result.columns[_column_index].columnDesc.__isset.columnScale) { - *str_slot = _desc_result.columns[_column_index].columnDesc.columnScale; - } else { - tuple->set_null(_tuple_desc->slots()[21]->null_indicator_offset()); + for (int i = 0; i < columns_num; ++i) { + if (_desc_result.columns[i].columnDesc.__isset.columnScale) { + int64_t src = _desc_result.columns[i].columnDesc.columnScale; + fill_dest_column(block, &src, _s_col_columns[21]); + } else { + fill_dest_column(block, nullptr, _s_col_columns[21]); + } } } // GENERATION_EXPRESSION - { tuple->set_null(_tuple_desc->slots()[22]->null_indicator_offset()); } - // SRS_ID - { tuple->set_null(_tuple_desc->slots()[23]->null_indicator_offset()); } - _column_index++; - return Status::OK(); -} - -Status SchemaColumnsScanner::get_new_desc() { - TDescribeTableParams desc_params; - desc_params.__set_db(_db_result.dbs[_db_index - 1]); - if (_db_result.__isset.catalogs) { - desc_params.__set_catalog(_db_result.catalogs[_db_index - 1]); - } - desc_params.__set_table_name(_table_result.tables[_table_index++]); - if (nullptr != _param->current_user_ident) { - desc_params.__set_current_user_ident(*(_param->current_user_ident)); - } else { - if (nullptr != _param->user) { - desc_params.__set_user(*(_param->user)); - } - if (nullptr != _param->user_ip) { - desc_params.__set_user_ip(*(_param->user_ip)); + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _s_col_columns[22]); } } - - if (nullptr != _param->ip && 0 != _param->port) { - RETURN_IF_ERROR(SchemaHelper::describe_table(*(_param->ip), _param->port, desc_params, - &_desc_result)); - } else { - return Status::InternalError("IP or port doesn't exists"); - } - _column_index = 0; - - return Status::OK(); -} - -Status SchemaColumnsScanner::get_new_table() { - TGetTablesParams table_params; - table_params.__set_db(_db_result.dbs[_db_index]); - if (_db_result.__isset.catalogs) { - table_params.__set_catalog(_db_result.catalogs[_db_index]); - } - _db_index++; - if (nullptr != _param->table) { - table_params.__set_pattern(*(_param->table)); - } - if (nullptr != _param->current_user_ident) { - table_params.__set_current_user_ident(*(_param->current_user_ident)); - } else { - if (nullptr != _param->user) { - table_params.__set_user(*(_param->user)); - } - if (nullptr != _param->user_ip) { - table_params.__set_user_ip(*(_param->user_ip)); + // SRS_ID + { + for (int i = 0; i < columns_num; ++i) { + fill_dest_column(block, nullptr, _s_col_columns[23]); } } - - if (nullptr != _param->ip && 0 != _param->port) { - RETURN_IF_ERROR(SchemaHelper::get_table_names(*(_param->ip), _param->port, table_params, - &_table_result)); - } else { - return Status::InternalError("IP or port doesn't exists"); - } - _table_index = 0; return Status::OK(); } -Status SchemaColumnsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("use this class before inited."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input parameter is nullptr."); - } - while (_column_index >= _desc_result.columns.size()) { - if (_table_index >= _table_result.tables.size()) { - if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); - } else { - *eos = true; - return Status::OK(); - } - } else { - RETURN_IF_ERROR(get_new_desc()); - } - } - - *eos = false; - return fill_one_row(tuple, pool); -} - } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.h b/be/src/exec/schema_scanner/schema_columns_scanner.h index 25afa984d6ed34..9ebe10b380d000 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.h +++ b/be/src/exec/schema_scanner/schema_columns_scanner.h @@ -27,25 +27,24 @@ namespace doris { class SchemaColumnsScanner : public SchemaScanner { public: SchemaColumnsScanner(); - virtual ~SchemaColumnsScanner(); - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + ~SchemaColumnsScanner() override; + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); - Status get_new_desc(); - Status get_create_table(std::string* result); - std::string to_mysql_data_type_string(TColumnDesc& desc); - std::string type_to_string(TColumnDesc& desc); + Status _get_new_table(); + Status _get_new_desc(); + Status _get_create_table(std::string* result); + Status _fill_block_impl(vectorized::Block* block); + std::string _to_mysql_data_type_string(TColumnDesc& desc); + std::string _type_to_string(TColumnDesc& desc); int _db_index; int _table_index; - int _column_index; TGetDbsResult _db_result; TGetTablesResult _table_result; TDescribeTableResult _desc_result; - static SchemaScanner::ColumnDesc _s_col_columns[]; + static std::vector _s_col_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp index 665947fdd839ef..b85885571837e1 100644 --- a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp @@ -18,12 +18,12 @@ #include "schema_dummy_scanner.h" namespace { -doris::SchemaScanner::ColumnDesc DUMMY_COLUMN; +std::vector DUMMY_COLUMN; } namespace doris { -SchemaDummyScanner::SchemaDummyScanner() : SchemaScanner(&DUMMY_COLUMN, 0) {} +SchemaDummyScanner::SchemaDummyScanner() : SchemaScanner(DUMMY_COLUMN) {} SchemaDummyScanner::~SchemaDummyScanner() {} @@ -31,7 +31,7 @@ Status SchemaDummyScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaDummyScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaDummyScanner::get_next_block(vectorized::Block* block, bool* eos) { *eos = true; return Status::OK(); } diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.h b/be/src/exec/schema_scanner/schema_dummy_scanner.h index 2b661ae58bad24..d48466e83a0827 100644 --- a/be/src/exec/schema_scanner/schema_dummy_scanner.h +++ b/be/src/exec/schema_scanner/schema_dummy_scanner.h @@ -24,9 +24,9 @@ namespace doris { class SchemaDummyScanner : public SchemaScanner { public: SchemaDummyScanner(); - virtual ~SchemaDummyScanner(); - virtual Status start(RuntimeState* state = nullptr); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + ~SchemaDummyScanner() override; + Status start(RuntimeState* state = nullptr) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_files_scanner.cpp b/be/src/exec/schema_scanner/schema_files_scanner.cpp index c9a6b9721d700b..762db774f96eb5 100644 --- a/be/src/exec/schema_scanner/schema_files_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_files_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaFilesScanner::_s_tbls_columns[] = { +std::vector SchemaFilesScanner::_s_tbls_columns = { // name, type, size, is_null {"FILE_ID", TYPE_BIGINT, sizeof(int64_t), true}, {"FILE_NAME", TYPE_STRING, sizeof(StringRef), true}, @@ -66,8 +66,7 @@ SchemaScanner::ColumnDesc SchemaFilesScanner::_s_tbls_columns[] = { }; SchemaFilesScanner::SchemaFilesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_FILES), _db_index(0), _table_index(0) {} @@ -78,24 +77,24 @@ Status SchemaFilesScanner::start(RuntimeState* state) { return Status::InternalError("used before initialized."); } TGetDbsParams db_params; - if (NULL != _param->db) { + if (nullptr != _param->db) { db_params.__set_pattern(*(_param->db)); } if (nullptr != _param->catalog) { db_params.__set_catalog(*(_param->catalog)); } - if (NULL != _param->current_user_ident) { + if (nullptr != _param->current_user_ident) { db_params.__set_current_user_ident(*(_param->current_user_ident)); } else { - if (NULL != _param->user) { + if (nullptr != _param->user) { db_params.__set_user(*(_param->user)); } - if (NULL != _param->user_ip) { + if (nullptr != _param->user_ip) { db_params.__set_user_ip(*(_param->user_ip)); } } - if (NULL != _param->ip && 0 != _param->port) { + if (nullptr != _param->ip && 0 != _param->port) { RETURN_IF_ERROR( SchemaHelper::get_db_names(*(_param->ip), _param->port, db_params, &_db_result)); } else { @@ -104,11 +103,11 @@ Status SchemaFilesScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaFilesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaFilesScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } *eos = true; diff --git a/be/src/exec/schema_scanner/schema_files_scanner.h b/be/src/exec/schema_scanner/schema_files_scanner.h index 012d1ed097c6a1..0e33bf6dd32e00 100644 --- a/be/src/exec/schema_scanner/schema_files_scanner.h +++ b/be/src/exec/schema_scanner/schema_files_scanner.h @@ -25,16 +25,16 @@ namespace doris { class SchemaFilesScanner : public SchemaScanner { public: SchemaFilesScanner(); - virtual ~SchemaFilesScanner(); + ~SchemaFilesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; int _db_index; int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp index 71c10ce1939b3d..41ba26da4b6e23 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp @@ -24,7 +24,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaPartitionsScanner::_s_tbls_columns[] = { +std::vector SchemaPartitionsScanner::_s_tbls_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), true}, @@ -54,8 +54,7 @@ SchemaScanner::ColumnDesc SchemaPartitionsScanner::_s_tbls_columns[] = { }; SchemaPartitionsScanner::SchemaPartitionsScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_PARTITIONS), _db_index(0), _table_index(0) {} @@ -66,24 +65,24 @@ Status SchemaPartitionsScanner::start(RuntimeState* state) { return Status::InternalError("used before initialized."); } TGetDbsParams db_params; - if (NULL != _param->db) { + if (nullptr != _param->db) { db_params.__set_pattern(*(_param->db)); } if (nullptr != _param->catalog) { db_params.__set_catalog(*(_param->catalog)); } - if (NULL != _param->current_user_ident) { + if (nullptr != _param->current_user_ident) { db_params.__set_current_user_ident(*(_param->current_user_ident)); } else { - if (NULL != _param->user) { + if (nullptr != _param->user) { db_params.__set_user(*(_param->user)); } - if (NULL != _param->user_ip) { + if (nullptr != _param->user_ip) { db_params.__set_user_ip(*(_param->user_ip)); } } - if (NULL != _param->ip && 0 != _param->port) { + if (nullptr != _param->ip && 0 != _param->port) { RETURN_IF_ERROR( SchemaHelper::get_db_names(*(_param->ip), _param->port, db_params, &_db_result)); } else { @@ -92,11 +91,11 @@ Status SchemaPartitionsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaPartitionsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaPartitionsScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } *eos = true; diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.h b/be/src/exec/schema_scanner/schema_partitions_scanner.h index 5bce10cfbbaa17..197f917a1de4f1 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.h +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.h @@ -25,16 +25,16 @@ namespace doris { class SchemaPartitionsScanner : public SchemaScanner { public: SchemaPartitionsScanner(); - virtual ~SchemaPartitionsScanner(); + ~SchemaPartitionsScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; int _db_index; int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp index 800be0c1aad2a6..2744531bd775f6 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp @@ -31,7 +31,7 @@ #include "runtime/primitive_type.h" #include "vec/common/string_ref.h" namespace doris { -SchemaScanner::ColumnDesc SchemaRowsetsScanner::_s_tbls_columns[] = { +std::vector SchemaRowsetsScanner::_s_tbls_columns = { // name, type, size, is_null {"BACKEND_ID", TYPE_BIGINT, sizeof(int64_t), true}, {"ROWSET_ID", TYPE_VARCHAR, sizeof(StringRef), true}, @@ -50,36 +50,20 @@ SchemaScanner::ColumnDesc SchemaRowsetsScanner::_s_tbls_columns[] = { }; SchemaRowsetsScanner::SchemaRowsetsScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_ROWSETS), backend_id_(0), - rowsets_idx_(0) {}; + _rowsets_idx(0) {}; Status SchemaRowsetsScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } backend_id_ = state->backend_id(); - RETURN_IF_ERROR(get_all_rowsets()); + RETURN_IF_ERROR(_get_all_rowsets()); return Status::OK(); } -Status SchemaRowsetsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("Used before initialized."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input pointer is nullptr."); - } - if (rowsets_idx_ >= rowsets_.size()) { - *eos = true; - return Status::OK(); - } - *eos = false; - return fill_one_row(tuple, pool); -} - -Status SchemaRowsetsScanner::get_all_rowsets() { +Status SchemaRowsetsScanner::_get_all_rowsets() { std::vector tablets = StorageEngine::instance()->tablet_manager()->get_all_tablet(); for (const auto& tablet : tablets) { @@ -97,80 +81,131 @@ Status SchemaRowsetsScanner::get_all_rowsets() { return Status::OK(); } -Status SchemaRowsetsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - RowsetSharedPtr rowset = rowsets_[rowsets_idx_]; +Status SchemaRowsetsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + if (_rowsets_idx >= rowsets_.size()) { + *eos = true; + return Status::OK(); + } + *eos = false; + return _fill_block_impl(block); +} + +Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { + size_t fill_rowsets_num = std::min(1000ul, rowsets_.size() - _rowsets_idx); + auto fill_idx_begin = _rowsets_idx; + auto fill_idx_end = _rowsets_idx + fill_rowsets_num; // BACKEND_ID { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - *(reinterpret_cast(slot)) = backend_id_; + int64_t src = backend_id_; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + fill_dest_column(block, &src, _s_tbls_columns[0]); + } } // ROWSET_ID { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string rowset_id = rowset->rowset_id().to_string(); - str_slot->data = (char*)pool->allocate(rowset_id.size()); - str_slot->size = rowset_id.size(); - memcpy(const_cast(str_slot->data), rowset_id.c_str(), str_slot->size); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + std::string rowset_id = rowset->rowset_id().to_string(); + StringRef str = StringRef(rowset_id.c_str(), rowset_id.size()); + fill_dest_column(block, &str, _s_tbls_columns[1]); + } } // TABLET_ID { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->rowset_meta()->tablet_id(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->rowset_meta()->tablet_id(); + fill_dest_column(block, &src, _s_tbls_columns[2]); + } } // ROWSET_NUM_ROWS { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->num_rows(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->num_rows(); + fill_dest_column(block, &src, _s_tbls_columns[3]); + } } // TXN_ID { - void* slot = tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->txn_id(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->txn_id(); + fill_dest_column(block, &src, _s_tbls_columns[4]); + } } // NUM_SEGMENTS { - void* slot = tuple->get_slot(_tuple_desc->slots()[5]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->num_segments(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->num_segments(); + fill_dest_column(block, &src, _s_tbls_columns[5]); + } } // START_VERSION { - void* slot = tuple->get_slot(_tuple_desc->slots()[6]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->start_version(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->start_version(); + fill_dest_column(block, &src, _s_tbls_columns[6]); + } } // END_VERSION { - void* slot = tuple->get_slot(_tuple_desc->slots()[7]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->end_version(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + int64_t src = rowset->end_version(); + fill_dest_column(block, &src, _s_tbls_columns[7]); + } } // INDEX_DISK_SIZE { - void* slot = tuple->get_slot(_tuple_desc->slots()[8]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->index_disk_size(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + size_t src = rowset->index_disk_size(); + fill_dest_column(block, &src, _s_tbls_columns[8]); + } } // DATA_DISK_SIZE { - void* slot = tuple->get_slot(_tuple_desc->slots()[9]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->data_disk_size(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + size_t src = rowset->data_disk_size(); + fill_dest_column(block, &src, _s_tbls_columns[9]); + } } // CREATION_TIME { - void* slot = tuple->get_slot(_tuple_desc->slots()[10]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->creation_time(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + size_t src = rowset->creation_time(); + fill_dest_column(block, &src, _s_tbls_columns[10]); + } } // OLDEST_WRITE_TIMESTAMP { - void* slot = tuple->get_slot(_tuple_desc->slots()[11]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->oldest_write_timestamp(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + size_t src = rowset->oldest_write_timestamp(); + fill_dest_column(block, &src, _s_tbls_columns[11]); + } } // NEWEST_WRITE_TIMESTAMP { - void* slot = tuple->get_slot(_tuple_desc->slots()[12]->tuple_offset()); - *(reinterpret_cast(slot)) = rowset->newest_write_timestamp(); + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + size_t src = rowset->newest_write_timestamp(); + fill_dest_column(block, &src, _s_tbls_columns[12]); + } } - ++rowsets_idx_; + _rowsets_idx += fill_rowsets_num; return Status::OK(); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.h b/be/src/exec/schema_scanner/schema_rowsets_scanner.h index dd642225fad51b..5869f97cdcd655 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include @@ -33,18 +34,15 @@ class SchemaRowsetsScanner : public SchemaScanner { ~SchemaRowsetsScanner() override = default; Status start(RuntimeState* state) override; - Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_all_rowsets(); - // Status get_new_segments(); - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _get_all_rowsets(); + Status _fill_block_impl(vectorized::Block* block); -private: - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; int64_t backend_id_ = 0; + size_t _rowsets_idx = 0; std::vector rowsets_; - // used for traversing rowsets_ - int rowsets_idx_ = 0; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp index 6b082ba82af755..03072073d24d10 100644 --- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaSchemaPrivilegesScanner::_s_tbls_columns[] = { +std::vector SchemaSchemaPrivilegesScanner::_s_tbls_columns = { // name, type, size, is_null {"GRANTEE", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, @@ -33,9 +33,7 @@ SchemaScanner::ColumnDesc SchemaSchemaPrivilegesScanner::_s_tbls_columns[] = { }; SchemaSchemaPrivilegesScanner::SchemaSchemaPrivilegesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _priv_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_SCHEMA_PRIVILEGES) {} SchemaSchemaPrivilegesScanner::~SchemaSchemaPrivilegesScanner() {} @@ -43,76 +41,11 @@ Status SchemaSchemaPrivilegesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); return Status::OK(); } -Status SchemaSchemaPrivilegesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - const TPrivilegeStatus& priv_status = _priv_result.privileges[_priv_index]; - // grantee - { - Status status = fill_one_col(&priv_status.grantee, pool, - tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // catalog - // This value is always def. - { - std::string definer = "def"; - Status status = fill_one_col(&definer, pool, - tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // schema - { - Status status = fill_one_col(&priv_status.schema, pool, - tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // privilege type - { - Status status = fill_one_col(&priv_status.privilege_type, pool, - tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // is grantable - { - Status status = fill_one_col(&priv_status.is_grantable, pool, - tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - _priv_index++; - return Status::OK(); -} - -Status SchemaSchemaPrivilegesScanner::fill_one_col(const std::string* src, MemPool* pool, - void* slot) { - if (nullptr == slot || nullptr == pool || nullptr == src) { - return Status::InternalError("input pointer is nullptr."); - } - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - return Status::OK(); -} - -Status SchemaSchemaPrivilegesScanner::get_new_table() { +Status SchemaSchemaPrivilegesScanner::_get_new_table() { TGetTablesParams table_params; if (nullptr != _param->wild) { table_params.__set_pattern(*(_param->wild)); @@ -134,23 +67,71 @@ Status SchemaSchemaPrivilegesScanner::get_new_table() { } else { return Status::InternalError("IP or port doesn't exists"); } - _priv_index = 0; return Status::OK(); } -Status SchemaSchemaPrivilegesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaSchemaPrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - if (_priv_index >= _priv_result.privileges.size()) { - *eos = true; + + *eos = true; + if (!_priv_result.privileges.size()) { return Status::OK(); } - *eos = false; - return fill_one_row(tuple, pool); + return _fill_block_impl(block); +} + +Status SchemaSchemaPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { + auto privileges_num = _priv_result.privileges.size(); + + // grantee + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); + fill_dest_column(block, &str, _s_tbls_columns[0]); + } + } + // catalog + // This value is always def. + { + std::string definer = "def"; + StringRef str = StringRef(definer.c_str(), definer.size()); + for (int i = 0; i < privileges_num; ++i) { + fill_dest_column(block, &str, _s_tbls_columns[1]); + } + } + // schema + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.schema.c_str(), priv_status.schema.size()); + fill_dest_column(block, &str, _s_tbls_columns[2]); + } + } + // privilege type + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.privilege_type.c_str(), + priv_status.privilege_type.size()); + fill_dest_column(block, &str, _s_tbls_columns[3]); + } + } + // is grantable + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = + StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); + fill_dest_column(block, &str, _s_tbls_columns[4]); + } + } + return Status::OK(); } } // namespace doris \ No newline at end of file diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h index d2f7c0f9b6cd42..a52643afd12fb1 100644 --- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h @@ -25,19 +25,17 @@ namespace doris { class SchemaSchemaPrivilegesScanner : public SchemaScanner { public: SchemaSchemaPrivilegesScanner(); - virtual ~SchemaSchemaPrivilegesScanner(); + ~SchemaSchemaPrivilegesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); - Status fill_one_col(const std::string* src, MemPool* pool, void* slot); + Status _get_new_table(); + Status _fill_block_impl(vectorized::Block* block); - int _priv_index; TListPrivilegesResult _priv_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp index 10427963f9089b..647a19a47aa2ed 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaSchemataScanner::_s_columns[] = { +std::vector SchemaSchemataScanner::_s_columns = { // name, type, size {"CATALOG_NAME", TYPE_VARCHAR, sizeof(StringRef), true}, {"SCHEMA_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -33,8 +33,7 @@ SchemaScanner::ColumnDesc SchemaSchemataScanner::_s_columns[] = { }; SchemaSchemataScanner::SchemaSchemataScanner() - : SchemaScanner(_s_columns, sizeof(_s_columns) / sizeof(SchemaScanner::ColumnDesc)), - _db_index(0) {} + : SchemaScanner(_s_columns, TSchemaTableType::SCH_SCHEMATA) {} SchemaSchemataScanner::~SchemaSchemataScanner() = default; @@ -70,73 +69,67 @@ Status SchemaSchemataScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaSchemataScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); +Status SchemaSchemataScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before Initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + *eos = true; + if (!_db_result.dbs.size()) { + return Status::OK(); + } + return _fill_block_impl(block); +} + +Status SchemaSchemataScanner::_fill_block_impl(vectorized::Block* block) { + auto dbs_num = _db_result.dbs.size(); // catalog { - if (!_db_result.__isset.catalogs) { - tuple->set_null(_tuple_desc->slots()[0]->null_indicator_offset()); - } else { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string catalog_name = _db_result.catalogs[_db_index]; - str_slot->data = (char*)pool->allocate(catalog_name.size()); - str_slot->size = catalog_name.size(); - memcpy(const_cast(str_slot->data), catalog_name.c_str(), str_slot->size); + for (int i = 0; i < dbs_num; ++i) { + if (!_db_result.__isset.catalogs) { + fill_dest_column(block, nullptr, _s_columns[0]); + } else { + std::string catalog_name = _db_result.catalogs[i]; + StringRef str = StringRef(catalog_name.c_str(), catalog_name.size()); + fill_dest_column(block, &str, _s_columns[0]); + } } } // schema { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index]); - str_slot->data = (char*)pool->allocate(db_name.size()); - str_slot->size = db_name.size(); - memcpy(const_cast(str_slot->data), db_name.c_str(), str_slot->size); + for (int i = 0; i < dbs_num; ++i) { + std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[i]); + StringRef str = StringRef(db_name.c_str(), db_name.size()); + fill_dest_column(block, &str, _s_columns[1]); + } } // DEFAULT_CHARACTER_SET_NAME { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = strlen("utf8") + 1; - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); + for (int i = 0; i < dbs_num; ++i) { + std::string src = "utf8"; + StringRef str = StringRef(src.c_str(), src.size()); + fill_dest_column(block, &str, _s_columns[2]); } - memcpy(const_cast(str_slot->data), "utf8", str_slot->size); } // DEFAULT_COLLATION_NAME { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = strlen("utf8_general_ci") + 1; - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); + for (int i = 0; i < dbs_num; ++i) { + std::string src = "utf8_general_ci"; + StringRef str = StringRef(src.c_str(), src.size()); + fill_dest_column(block, &str, _s_columns[3]); } - memcpy(const_cast(str_slot->data), "utf8_general_ci", str_slot->size); } // SQL_PATH - { tuple->set_null(_tuple_desc->slots()[4]->null_indicator_offset()); } - _db_index++; - return Status::OK(); -} - -Status SchemaSchemataScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("Used before Initialized."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input pointer is nullptr."); - } - if (_db_index >= _db_result.dbs.size()) { - *eos = true; - return Status::OK(); + { + for (int i = 0; i < dbs_num; ++i) { + fill_dest_column(block, nullptr, _s_columns[4]); + } } - *eos = false; - return fill_one_row(tuple, pool); + return Status::OK(); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.h b/be/src/exec/schema_scanner/schema_schemata_scanner.h index f6bc752021aaf0..572f8b741df23f 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.h +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.h @@ -25,17 +25,16 @@ namespace doris { class SchemaSchemataScanner : public SchemaScanner { public: SchemaSchemataScanner(); - virtual ~SchemaSchemataScanner(); + ~SchemaSchemataScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); - int _db_index; TGetDbsResult _db_result; - static SchemaScanner::ColumnDesc _s_columns[]; + static std::vector _s_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_statistics_scanner.cpp b/be/src/exec/schema_scanner/schema_statistics_scanner.cpp index 366765dda604f3..ed2e713da03aed 100644 --- a/be/src/exec/schema_scanner/schema_statistics_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_statistics_scanner.cpp @@ -22,7 +22,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaStatisticsScanner::_s_cols_statistics[] = { +std::vector SchemaStatisticsScanner::_s_cols_statistics = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -41,9 +41,7 @@ SchemaScanner::ColumnDesc SchemaStatisticsScanner::_s_cols_statistics[] = { {"COMMENT", TYPE_VARCHAR, sizeof(StringRef), true}, }; -SchemaStatisticsScanner::SchemaStatisticsScanner() - : SchemaScanner(_s_cols_statistics, - sizeof(_s_cols_statistics) / sizeof(SchemaScanner::ColumnDesc)) {} +SchemaStatisticsScanner::SchemaStatisticsScanner() : SchemaScanner(_s_cols_statistics) {} SchemaStatisticsScanner::~SchemaStatisticsScanner() {} diff --git a/be/src/exec/schema_scanner/schema_statistics_scanner.h b/be/src/exec/schema_scanner/schema_statistics_scanner.h index 7953a57987132c..e31c27277b2de2 100644 --- a/be/src/exec/schema_scanner/schema_statistics_scanner.h +++ b/be/src/exec/schema_scanner/schema_statistics_scanner.h @@ -23,9 +23,9 @@ namespace doris { class SchemaStatisticsScanner : public SchemaScanner { public: SchemaStatisticsScanner(); - virtual ~SchemaStatisticsScanner(); + ~SchemaStatisticsScanner() override; private: - static SchemaScanner::ColumnDesc _s_cols_statistics[]; + static std::vector _s_cols_statistics; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp index 1d572bc27650d3..d959145168c66c 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp @@ -17,13 +17,15 @@ #include "exec/schema_scanner/schema_table_privileges_scanner.h" +#include + #include "exec/schema_scanner/schema_helper.h" #include "runtime/primitive_type.h" #include "vec/common/string_ref.h" namespace doris { -SchemaScanner::ColumnDesc SchemaTablePrivilegesScanner::_s_tbls_columns[] = { +std::vector SchemaTablePrivilegesScanner::_s_tbls_columns = { // name, type, size, is_null {"GRANTEE", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, @@ -34,9 +36,7 @@ SchemaScanner::ColumnDesc SchemaTablePrivilegesScanner::_s_tbls_columns[] = { }; SchemaTablePrivilegesScanner::SchemaTablePrivilegesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _priv_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_TABLE_PRIVILEGES) {} SchemaTablePrivilegesScanner::~SchemaTablePrivilegesScanner() {} @@ -44,84 +44,11 @@ Status SchemaTablePrivilegesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } - RETURN_IF_ERROR(get_new_table()); - return Status::OK(); -} - -Status SchemaTablePrivilegesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - const TPrivilegeStatus& priv_status = _priv_result.privileges[_priv_index]; - // grantee - { - Status status = fill_one_col(&priv_status.grantee, pool, - tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // catalog - // This value is always def. - { - std::string definer = "def"; - Status status = fill_one_col(&definer, pool, - tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // schema - { - Status status = fill_one_col(&priv_status.schema, pool, - tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // table name - { - Status status = fill_one_col(&priv_status.table_name, pool, - tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // privilege type - { - Status status = fill_one_col(&priv_status.privilege_type, pool, - tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // is grantable - { - Status status = fill_one_col(&priv_status.is_grantable, pool, - tuple->get_slot(_tuple_desc->slots()[5]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - _priv_index++; + RETURN_IF_ERROR(_get_new_table()); return Status::OK(); } -Status SchemaTablePrivilegesScanner::fill_one_col(const std::string* src, MemPool* pool, - void* slot) { - if (nullptr == slot || nullptr == pool || nullptr == src) { - return Status::InternalError("input pointer is nullptr."); - } - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - return Status::OK(); -} - -Status SchemaTablePrivilegesScanner::get_new_table() { +Status SchemaTablePrivilegesScanner::_get_new_table() { TGetTablesParams table_params; if (nullptr != _param->wild) { table_params.__set_pattern(*(_param->wild)); @@ -143,23 +70,80 @@ Status SchemaTablePrivilegesScanner::get_new_table() { } else { return Status::InternalError("IP or port doesn't exists"); } - _priv_index = 0; return Status::OK(); } -Status SchemaTablePrivilegesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaTablePrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - if (_priv_index >= _priv_result.privileges.size()) { - *eos = true; + + *eos = true; + if (!_priv_result.privileges.size()) { return Status::OK(); } - *eos = false; - return fill_one_row(tuple, pool); + return _fill_block_impl(block); +} + +Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block* block) { + auto privileges_num = _priv_result.privileges.size(); + + // grantee + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); + fill_dest_column(block, &str, _s_tbls_columns[0]); + } + } + // catalog + // This value is always def. + { + std::string definer = "def"; + StringRef str = StringRef(definer.c_str(), definer.size()); + for (int i = 0; i < privileges_num; ++i) { + fill_dest_column(block, &str, _s_tbls_columns[1]); + } + } + // schema + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.schema.c_str(), priv_status.schema.size()); + fill_dest_column(block, &str, _s_tbls_columns[2]); + } + } + // table name + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = + StringRef(priv_status.table_name.c_str(), priv_status.table_name.size()); + fill_dest_column(block, &str, _s_tbls_columns[3]); + } + } + // privilege type + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.privilege_type.c_str(), + priv_status.privilege_type.size()); + fill_dest_column(block, &str, _s_tbls_columns[4]); + } + } + // is grantable + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = + StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); + fill_dest_column(block, &str, _s_tbls_columns[5]); + } + } + return Status::OK(); } } // namespace doris \ No newline at end of file diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h index d14e8005220832..00545717c22ea0 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h @@ -25,19 +25,17 @@ namespace doris { class SchemaTablePrivilegesScanner : public SchemaScanner { public: SchemaTablePrivilegesScanner(); - virtual ~SchemaTablePrivilegesScanner(); + ~SchemaTablePrivilegesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); - Status fill_one_col(const std::string* src, MemPool* pool, void* slot); + Status _get_new_table(); + Status _fill_block_impl(vectorized::Block* block); - int _priv_index; TListPrivilegesResult _priv_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index ee9febe3f201db..054897b516b202 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -17,13 +17,15 @@ #include "exec/schema_scanner/schema_tables_scanner.h" +#include "common/status.h" #include "exec/schema_scanner/schema_helper.h" #include "runtime/primitive_type.h" +#include "vec/columns/column_complex.h" #include "vec/common/string_ref.h" namespace doris { -SchemaScanner::ColumnDesc SchemaTablesScanner::_s_tbls_columns[] = { +std::vector SchemaTablesScanner::_s_tbls_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -49,10 +51,7 @@ SchemaScanner::ColumnDesc SchemaTablesScanner::_s_tbls_columns[] = { }; SchemaTablesScanner::SchemaTablesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _db_index(0), - _table_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_TABLES), _db_index(0) {} SchemaTablesScanner::~SchemaTablesScanner() {} @@ -87,224 +86,225 @@ Status SchemaTablesScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaTablesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - const TTableStatus& tbl_status = _table_result.tables[_table_index]; +Status SchemaTablesScanner::_get_new_table() { + TGetTablesParams table_params; + table_params.__set_db(_db_result.dbs[_db_index]); + if (_db_result.__isset.catalogs) { + table_params.__set_catalog(_db_result.catalogs[_db_index]); + } + _db_index++; + if (nullptr != _param->wild) { + table_params.__set_pattern(*(_param->wild)); + } + if (nullptr != _param->current_user_ident) { + table_params.__set_current_user_ident(*(_param->current_user_ident)); + } else { + if (nullptr != _param->user) { + table_params.__set_user(*(_param->user)); + } + if (nullptr != _param->user_ip) { + table_params.__set_user_ip(*(_param->user_ip)); + } + } + + if (nullptr != _param->ip && 0 != _param->port) { + RETURN_IF_ERROR(SchemaHelper::list_table_status(*(_param->ip), _param->port, table_params, + &_table_result)); + } else { + return Status::InternalError("IP or port doesn't exists"); + } + return Status::OK(); +} + +Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { + auto table_num = _table_result.tables.size(); // catalog - { - if (!_db_result.__isset.catalogs) { - tuple->set_null(_tuple_desc->slots()[0]->null_indicator_offset()); - } else { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - std::string catalog_name = _db_result.catalogs[_db_index - 1]; - str_slot->data = (char*)pool->allocate(catalog_name.size()); - str_slot->size = catalog_name.size(); - memcpy(const_cast(str_slot->data), catalog_name.c_str(), str_slot->size); + if (_db_result.__isset.catalogs) { + std::string catalog_name = _db_result.catalogs[_db_index - 1]; + StringRef str_slot = StringRef(catalog_name.c_str(), catalog_name.size()); + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, &str_slot, _s_tbls_columns[0]); + } + } else { + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[0]); } } // schema { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); - str_slot->data = (char*)pool->allocate(db_name.size()); - str_slot->size = db_name.size(); - memcpy(const_cast(str_slot->data), db_name.c_str(), str_slot->size); + StringRef str_slot = StringRef(db_name.c_str(), db_name.size()); + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, &str_slot, _s_tbls_columns[1]); + } } // name - { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.name; - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); + for (int i = 0; i < table_num; ++i) { + const std::string* src = &_table_result.tables[i].name; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _s_tbls_columns[2]); } // type - { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.type; - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); + for (int i = 0; i < table_num; ++i) { + const std::string* src = &_table_result.tables[i].type; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _s_tbls_columns[3]); } // engine - if (tbl_status.__isset.engine) { - void* slot = tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.engine; - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.engine) { + const std::string* src = &tbl_status.engine; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _s_tbls_columns[4]); + } else { + fill_dest_column(block, nullptr, _s_tbls_columns[4]); } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - } else { - tuple->set_null(_tuple_desc->slots()[4]->null_indicator_offset()); } // version - { tuple->set_null(_tuple_desc->slots()[5]->null_indicator_offset()); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[5]); + } // row_format - { tuple->set_null(_tuple_desc->slots()[6]->null_indicator_offset()); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[6]); + } // rows - if (tbl_status.__isset.rows) { - void* slot = tuple->get_slot(_tuple_desc->slots()[7]->tuple_offset()); - *(reinterpret_cast(slot)) = tbl_status.rows; - } else { - tuple->set_null(_tuple_desc->slots()[7]->null_indicator_offset()); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.rows) { + int64_t src = tbl_status.rows; + fill_dest_column(block, &src, _s_tbls_columns[7]); + } else { + fill_dest_column(block, nullptr, _s_tbls_columns[7]); + } } // avg_row_length - if (tbl_status.__isset.avg_row_length) { - void* slot = tuple->get_slot(_tuple_desc->slots()[8]->tuple_offset()); - *(reinterpret_cast(slot)) = tbl_status.avg_row_length; - } else { - tuple->set_null(_tuple_desc->slots()[8]->null_indicator_offset()); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.avg_row_length) { + int64_t src = tbl_status.avg_row_length; + fill_dest_column(block, &src, _s_tbls_columns[8]); + } else { + fill_dest_column(block, nullptr, _s_tbls_columns[8]); + } } // data_length - if (tbl_status.__isset.avg_row_length) { - void* slot = tuple->get_slot(_tuple_desc->slots()[9]->tuple_offset()); - *(reinterpret_cast(slot)) = tbl_status.data_length; - } else { - tuple->set_null(_tuple_desc->slots()[9]->null_indicator_offset()); - } // max_data_length - { tuple->set_null(_tuple_desc->slots()[10]->null_indicator_offset()); } + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.avg_row_length) { + int64_t src = tbl_status.data_length; + fill_dest_column(block, &src, _s_tbls_columns[9]); + } else { + fill_dest_column(block, nullptr, _s_tbls_columns[9]); + } + } + // max_data_length + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[10]); + } // index_length - { tuple->set_null(_tuple_desc->slots()[11]->null_indicator_offset()); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[11]); + } // data_free - { tuple->set_null(_tuple_desc->slots()[12]->null_indicator_offset()); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[12]); + } // auto_increment - { tuple->set_null(_tuple_desc->slots()[13]->null_indicator_offset()); } + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[13]); + } // creation_time - if (tbl_status.__isset.create_time) { - int64_t create_time = tbl_status.create_time; - if (create_time <= 0) { - tuple->set_null(_tuple_desc->slots()[14]->null_indicator_offset()); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.create_time) { + int64_t create_time = tbl_status.create_time; + if (create_time <= 0) { + fill_dest_column(block, nullptr, _s_tbls_columns[14]); + } else { + DateTimeValue time_slot; + time_slot.from_unixtime(create_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, &time_slot, _s_tbls_columns[14]); + } } else { - tuple->set_not_null(_tuple_desc->slots()[14]->null_indicator_offset()); - void* slot = tuple->get_slot(_tuple_desc->slots()[14]->tuple_offset()); - DateTimeValue* time_slot = reinterpret_cast(slot); - time_slot->from_unixtime(create_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, nullptr, _s_tbls_columns[14]); } } // update_time - if (tbl_status.__isset.update_time) { - int64_t update_time = tbl_status.update_time; - if (update_time <= 0) { - tuple->set_null(_tuple_desc->slots()[15]->null_indicator_offset()); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.update_time) { + int64_t update_time = tbl_status.update_time; + if (update_time <= 0) { + fill_dest_column(block, nullptr, _s_tbls_columns[15]); + } else { + DateTimeValue time_slot; + time_slot.from_unixtime(update_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, &time_slot, _s_tbls_columns[15]); + } } else { - tuple->set_not_null(_tuple_desc->slots()[15]->null_indicator_offset()); - void* slot = tuple->get_slot(_tuple_desc->slots()[15]->tuple_offset()); - DateTimeValue* time_slot = reinterpret_cast(slot); - time_slot->from_unixtime(update_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, nullptr, _s_tbls_columns[15]); } } // check_time - if (tbl_status.__isset.last_check_time) { - int64_t check_time = tbl_status.last_check_time; - if (check_time <= 0) { - tuple->set_null(_tuple_desc->slots()[16]->null_indicator_offset()); - } else { - tuple->set_not_null(_tuple_desc->slots()[16]->null_indicator_offset()); - void* slot = tuple->get_slot(_tuple_desc->slots()[16]->tuple_offset()); - DateTimeValue* time_slot = reinterpret_cast(slot); - time_slot->from_unixtime(check_time, TimezoneUtils::default_time_zone); + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.last_check_time) { + int64_t check_time = tbl_status.last_check_time; + if (check_time <= 0) { + fill_dest_column(block, nullptr, _s_tbls_columns[16]); + } else { + DateTimeValue time_slot; + time_slot.from_unixtime(check_time, TimezoneUtils::default_time_zone); + fill_dest_column(block, &time_slot, _s_tbls_columns[16]); + } } } // collation - if (tbl_status.__isset.collation) { - void* slot = tuple->get_slot(_tuple_desc->slots()[17]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.collation; - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - } else { - tuple->set_null(_tuple_desc->slots()[17]->null_indicator_offset()); - } - // checksum - { tuple->set_null(_tuple_desc->slots()[18]->null_indicator_offset()); } - // create_options - { tuple->set_null(_tuple_desc->slots()[19]->null_indicator_offset()); } - // create_comment - { - void* slot = tuple->get_slot(_tuple_desc->slots()[20]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.comment; - str_slot->size = src->length(); - if (str_slot->size == 0) { - str_slot->data = nullptr; + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.collation) { + const std::string* src = &tbl_status.collation; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _s_tbls_columns[17]); + } else { - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); + fill_dest_column(block, nullptr, _s_tbls_columns[17]); } } - _table_index++; - return Status::OK(); -} - -Status SchemaTablesScanner::get_new_table() { - TGetTablesParams table_params; - table_params.__set_db(_db_result.dbs[_db_index]); - if (_db_result.__isset.catalogs) { - table_params.__set_catalog(_db_result.catalogs[_db_index]); - } - _db_index++; - if (nullptr != _param->wild) { - table_params.__set_pattern(*(_param->wild)); + // checksum + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[18]); } - if (nullptr != _param->current_user_ident) { - table_params.__set_current_user_ident(*(_param->current_user_ident)); - } else { - if (nullptr != _param->user) { - table_params.__set_user(*(_param->user)); - } - if (nullptr != _param->user_ip) { - table_params.__set_user_ip(*(_param->user_ip)); - } + // create_options + for (int i = 0; i < table_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[19]); } - - if (nullptr != _param->ip && 0 != _param->port) { - RETURN_IF_ERROR(SchemaHelper::list_table_status(*(_param->ip), _param->port, table_params, - &_table_result)); - } else { - return Status::InternalError("IP or port doesn't exists"); + // create_comment + for (int i = 0; i < table_num; ++i) { + const std::string* src = &_table_result.tables[i].comment; + StringRef str_slot = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str_slot, _s_tbls_columns[20]); } - _table_index = 0; return Status::OK(); } -Status SchemaTablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaTablesScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - while (_table_index >= _table_result.tables.size()) { - if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); - } else { - *eos = true; - return Status::OK(); - } + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(_get_new_table()); + } else { + *eos = true; + return Status::OK(); } *eos = false; - return fill_one_row(tuple, pool); + return _fill_block_impl(block); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h index ff957999339eea..18488451ccf819 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_tables_scanner.h @@ -17,28 +17,29 @@ #pragma once +#include "common/status.h" #include "exec/schema_scanner.h" #include "gen_cpp/FrontendService_types.h" +#include "vec/core/block.h" namespace doris { class SchemaTablesScanner : public SchemaScanner { public: SchemaTablesScanner(); - virtual ~SchemaTablesScanner(); + ~SchemaTablesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _get_new_table(); + Status _fill_block_impl(vectorized::Block* block); int _db_index; - int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp index e2a1b86eb23e52..4fcdfecfbe8008 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaUserPrivilegesScanner::_s_tbls_columns[] = { +std::vector SchemaUserPrivilegesScanner::_s_tbls_columns = { // name, type, size, is_null {"GRANTEE", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, @@ -32,9 +32,7 @@ SchemaScanner::ColumnDesc SchemaUserPrivilegesScanner::_s_tbls_columns[] = { }; SchemaUserPrivilegesScanner::SchemaUserPrivilegesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _priv_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_USER_PRIVILEGES) {} SchemaUserPrivilegesScanner::~SchemaUserPrivilegesScanner() {} @@ -42,68 +40,11 @@ Status SchemaUserPrivilegesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); return Status::OK(); } -Status SchemaUserPrivilegesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - const TPrivilegeStatus& priv_status = _priv_result.privileges[_priv_index]; - // grantee - { - Status status = fill_one_col(&priv_status.grantee, pool, - tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // catalog - // This value is always def. - { - std::string definer = "def"; - Status status = fill_one_col(&definer, pool, - tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // privilege type - { - Status status = fill_one_col(&priv_status.privilege_type, pool, - tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - // is grantable - { - Status status = fill_one_col(&priv_status.is_grantable, pool, - tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset())); - if (!status.ok()) { - return status; - } - } - _priv_index++; - return Status::OK(); -} - -Status SchemaUserPrivilegesScanner::fill_one_col(const std::string* src, MemPool* pool, - void* slot) { - if (nullptr == slot || nullptr == pool || nullptr == src) { - return Status::InternalError("input pointer is nullptr."); - } - StringRef* str_slot = reinterpret_cast(slot); - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); - } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); - return Status::OK(); -} - -Status SchemaUserPrivilegesScanner::get_new_table() { +Status SchemaUserPrivilegesScanner::_get_new_table() { TGetTablesParams table_params; if (nullptr != _param->wild) { table_params.__set_pattern(*(_param->wild)); @@ -125,23 +66,63 @@ Status SchemaUserPrivilegesScanner::get_new_table() { } else { return Status::InternalError("IP or port doesn't exists"); } - _priv_index = 0; return Status::OK(); } -Status SchemaUserPrivilegesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { +Status SchemaUserPrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { + if (nullptr == block || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - if (_priv_index >= _priv_result.privileges.size()) { - *eos = true; + + *eos = true; + if (!_priv_result.privileges.size()) { return Status::OK(); } - *eos = false; - return fill_one_row(tuple, pool); + return _fill_block_impl(block); +} + +Status SchemaUserPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { + auto privileges_num = _priv_result.privileges.size(); + + // grantee + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); + fill_dest_column(block, &str, _s_tbls_columns[0]); + } + } + // catalog + // This value is always def. + { + std::string definer = "def"; + StringRef str = StringRef(definer.c_str(), definer.size()); + for (int i = 0; i < privileges_num; ++i) { + fill_dest_column(block, &str, _s_tbls_columns[1]); + } + } + // privilege type + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = StringRef(priv_status.privilege_type.c_str(), + priv_status.privilege_type.size()); + fill_dest_column(block, &str, _s_tbls_columns[2]); + } + } + // is grantable + { + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + StringRef str = + StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); + fill_dest_column(block, &str, _s_tbls_columns[3]); + } + } + return Status::OK(); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h index 2e2cbc80dafb61..1bef2b340b9e10 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h @@ -25,19 +25,17 @@ namespace doris { class SchemaUserPrivilegesScanner : public SchemaScanner { public: SchemaUserPrivilegesScanner(); - virtual ~SchemaUserPrivilegesScanner(); + ~SchemaUserPrivilegesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); - Status fill_one_col(const std::string* src, MemPool* pool, void* slot); + Status _get_new_table(); + Status _fill_block_impl(vectorized::Block* block); - int _priv_index; TListPrivilegesResult _priv_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.cpp b/be/src/exec/schema_scanner/schema_variables_scanner.cpp index f550e8a0dd2123..38fa4e0af191af 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_variables_scanner.cpp @@ -24,16 +24,14 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaVariablesScanner::_s_vars_columns[] = { +std::vector SchemaVariablesScanner::_s_vars_columns = { // name, type, size {"VARIABLE_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, {"VARIABLE_VALUE", TYPE_VARCHAR, sizeof(StringRef), false}, }; SchemaVariablesScanner::SchemaVariablesScanner(TVarType::type type) - : SchemaScanner(_s_vars_columns, - sizeof(_s_vars_columns) / sizeof(SchemaScanner::ColumnDesc)), - _type(type) {} + : SchemaScanner(_s_vars_columns, TSchemaTableType::SCH_VARIABLES), _type(type) {} SchemaVariablesScanner::~SchemaVariablesScanner() {} @@ -57,52 +55,40 @@ Status SchemaVariablesScanner::start(RuntimeState* state) { } else { return Status::InternalError("IP or port doesn't exists"); } - _begin = _var_result.variables.begin(); return Status::OK(); } -Status SchemaVariablesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { +Status SchemaVariablesScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("call this before initial."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("invalid parameter."); + } + + *eos = true; + if (_var_result.variables.empty()) { + return Status::OK(); + } + return _fill_block_impl(block); +} + +Status SchemaVariablesScanner::_fill_block_impl(vectorized::Block* block) { // variables names { - void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_begin->first.c_str()); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); + for (auto& it : _var_result.variables) { + StringRef str = StringRef(it.first.c_str(), it.first.size()); + fill_dest_column(block, &str, _s_vars_columns[0]); } - memcpy(const_cast(str_slot->data), _begin->first.c_str(), len + 1); - str_slot->size = len; } // value { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - int len = strlen(_begin->second.c_str()); - str_slot->data = (char*)pool->allocate(len + 1); - if (nullptr == str_slot->data) { - return Status::InternalError("No Memory."); + for (auto& it : _var_result.variables) { + StringRef str = StringRef(it.second.c_str(), it.second.size()); + fill_dest_column(block, &str, _s_vars_columns[1]); } - memcpy(const_cast(str_slot->data), _begin->second.c_str(), len + 1); - str_slot->size = len; } - ++_begin; return Status::OK(); } -Status SchemaVariablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("call this before initial."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("invalid parameter."); - } - if (_begin == _var_result.variables.end()) { - *eos = true; - return Status::OK(); - } - *eos = false; - return fill_one_row(tuple, pool); -} - } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.h b/be/src/exec/schema_scanner/schema_variables_scanner.h index 25daaf0e4f9c43..4befea7bda4803 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.h +++ b/be/src/exec/schema_scanner/schema_variables_scanner.h @@ -28,10 +28,10 @@ namespace doris { class SchemaVariablesScanner : public SchemaScanner { public: SchemaVariablesScanner(TVarType::type type); - virtual ~SchemaVariablesScanner(); + ~SchemaVariablesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: struct VariableStruct { @@ -39,13 +39,12 @@ class SchemaVariablesScanner : public SchemaScanner { const char* value; }; - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); - static SchemaScanner::ColumnDesc _s_vars_columns[]; + static std::vector _s_vars_columns; TShowVariableResult _var_result; TVarType::type _type; - std::map::iterator _begin; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_views_scanner.cpp b/be/src/exec/schema_scanner/schema_views_scanner.cpp index af04de20df6be3..fb64337cce70d4 100644 --- a/be/src/exec/schema_scanner/schema_views_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_views_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaViewsScanner::_s_tbls_columns[] = { +std::vector SchemaViewsScanner::_s_tbls_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringRef), false}, @@ -38,10 +38,7 @@ SchemaScanner::ColumnDesc SchemaViewsScanner::_s_tbls_columns[] = { }; SchemaViewsScanner::SchemaViewsScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _db_index(0), - _table_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_VIEWS), _db_index(0) {} SchemaViewsScanner::~SchemaViewsScanner() {} @@ -76,161 +73,136 @@ Status SchemaViewsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaViewsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { - // set all bit to not null - memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - const TTableStatus& tbl_status = _table_result.tables[_table_index]; +Status SchemaViewsScanner::_get_new_table() { + TGetTablesParams table_params; + table_params.__set_db(_db_result.dbs[_db_index++]); + if (nullptr != _param->wild) { + table_params.__set_pattern(*(_param->wild)); + } + if (nullptr != _param->current_user_ident) { + table_params.__set_current_user_ident(*(_param->current_user_ident)); + } else { + if (nullptr != _param->user) { + table_params.__set_user(*(_param->user)); + } + if (nullptr != _param->user_ip) { + table_params.__set_user_ip(*(_param->user_ip)); + } + } + table_params.__set_type("VIEW"); + + if (nullptr != _param->ip && 0 != _param->port) { + RETURN_IF_ERROR(SchemaHelper::list_table_status(*(_param->ip), _param->port, table_params, + &_table_result)); + } else { + return Status::InternalError("IP or port doesn't exists"); + } + return Status::OK(); +} + +Status SchemaViewsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(_get_new_table()); + } else { + *eos = true; + return Status::OK(); + } + *eos = false; + return _fill_block_impl(block); +} + +Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { + auto tables_num = _table_result.tables.size(); + // catalog - { tuple->set_null(_tuple_desc->slots()[0]->null_indicator_offset()); } + { + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[0]); + } + } // schema { - void* slot = tuple->get_slot(_tuple_desc->slots()[1]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); - str_slot->data = (char*)pool->allocate(db_name.size()); - str_slot->size = db_name.size(); - memcpy(const_cast(str_slot->data), db_name.c_str(), str_slot->size); + StringRef str = StringRef(db_name.c_str(), db_name.size()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _s_tbls_columns[1]); + } } // name { - void* slot = tuple->get_slot(_tuple_desc->slots()[2]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* src = &tbl_status.name; - str_slot->size = src->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memcpy failed."); + for (int i = 0; i < tables_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + const std::string* src = &tbl_status.name; + StringRef str = StringRef(src->c_str(), src->size()); + fill_dest_column(block, &str, _s_tbls_columns[2]); } - memcpy(const_cast(str_slot->data), src->c_str(), str_slot->size); } // definition { - void* slot = tuple->get_slot(_tuple_desc->slots()[3]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - const std::string* ddl_sql = &tbl_status.ddl_sql; - str_slot->size = ddl_sql->length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memcpy failed."); + for (int i = 0; i < tables_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + const std::string* src = &tbl_status.ddl_sql; + StringRef str = StringRef(src->c_str(), src->length()); + fill_dest_column(block, &str, _s_tbls_columns[3]); } - memcpy(const_cast(str_slot->data), ddl_sql->c_str(), str_slot->size); } // check_option { - void* slot = tuple->get_slot(_tuple_desc->slots()[4]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); - // This is from views in mysql const std::string check_option = "NONE"; - str_slot->size = check_option.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memcpy failed."); + StringRef str = StringRef(check_option.c_str(), check_option.length()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _s_tbls_columns[4]); } - memcpy(const_cast(str_slot->data), check_option.c_str(), str_slot->size); } // is_updatable { - void* slot = tuple->get_slot(_tuple_desc->slots()[5]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); // This is from views in mysql const std::string is_updatable = "NO"; - str_slot->size = is_updatable.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memcpy failed."); + StringRef str = StringRef(is_updatable.c_str(), is_updatable.length()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _s_tbls_columns[5]); } - memcpy(const_cast(str_slot->data), is_updatable.c_str(), str_slot->size); } // definer { - void* slot = tuple->get_slot(_tuple_desc->slots()[6]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); // This is from views in mysql const std::string definer = "root@%"; - str_slot->size = definer.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memcpy failed."); + StringRef str = StringRef(definer.c_str(), definer.length()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _s_tbls_columns[6]); } - memcpy(const_cast(str_slot->data), definer.c_str(), str_slot->size); } // security_type { - void* slot = tuple->get_slot(_tuple_desc->slots()[7]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); // This is from views in mysql const std::string security_type = "DEFINER"; - str_slot->size = security_type.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); + StringRef str = StringRef(security_type.c_str(), security_type.length()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _s_tbls_columns[7]); } - memcpy(const_cast(str_slot->data), security_type.c_str(), str_slot->size); } // character_set_client { - void* slot = tuple->get_slot(_tuple_desc->slots()[8]->tuple_offset()); - StringRef* str_slot = reinterpret_cast(slot); // This is from views in mysql const std::string encoding = "utf8"; - str_slot->size = encoding.length(); - str_slot->data = (char*)pool->allocate(str_slot->size); - if (nullptr == str_slot->data) { - return Status::InternalError("Allocate memory failed."); + StringRef str = StringRef(encoding.c_str(), encoding.length()); + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, &str, _s_tbls_columns[8]); } - memcpy(const_cast(str_slot->data), encoding.c_str(), str_slot->size); } // collation_connection - { tuple->set_null(_tuple_desc->slots()[9]->null_indicator_offset()); } - _table_index++; - return Status::OK(); -} - -Status SchemaViewsScanner::get_new_table() { - TGetTablesParams table_params; - table_params.__set_db(_db_result.dbs[_db_index++]); - if (nullptr != _param->wild) { - table_params.__set_pattern(*(_param->wild)); - } - if (nullptr != _param->current_user_ident) { - table_params.__set_current_user_ident(*(_param->current_user_ident)); - } else { - if (nullptr != _param->user) { - table_params.__set_user(*(_param->user)); - } - if (nullptr != _param->user_ip) { - table_params.__set_user_ip(*(_param->user_ip)); + { + for (int i = 0; i < tables_num; ++i) { + fill_dest_column(block, nullptr, _s_tbls_columns[9]); } } - table_params.__set_type("VIEW"); - - if (nullptr != _param->ip && 0 != _param->port) { - RETURN_IF_ERROR(SchemaHelper::list_table_status(*(_param->ip), _param->port, table_params, - &_table_result)); - } else { - return Status::InternalError("IP or port doesn't exists"); - } - _table_index = 0; return Status::OK(); } -Status SchemaViewsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { - if (!_is_init) { - return Status::InternalError("Used before initialized."); - } - if (nullptr == tuple || nullptr == pool || nullptr == eos) { - return Status::InternalError("input pointer is nullptr."); - } - while (_table_index >= _table_result.tables.size()) { - if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); - } else { - *eos = true; - return Status::OK(); - } - } - *eos = false; - return fill_one_row(tuple, pool); -} - } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_views_scanner.h b/be/src/exec/schema_scanner/schema_views_scanner.h index 3222b67e897d59..9eeca601f32e05 100644 --- a/be/src/exec/schema_scanner/schema_views_scanner.h +++ b/be/src/exec/schema_scanner/schema_views_scanner.h @@ -25,20 +25,19 @@ namespace doris { class SchemaViewsScanner : public SchemaScanner { public: SchemaViewsScanner(); - virtual ~SchemaViewsScanner(); + ~SchemaViewsScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _get_new_table(); + Status _fill_block_impl(vectorized::Block* block); int _db_index; - int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index 9d26ddf03b2b06..4b6eda4ba95b1f 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -17,14 +17,21 @@ #include "vec/exec/vschema_scan_node.h" +#include +#include + +#include "common/status.h" #include "exec/text_converter.h" #include "exec/text_converter.hpp" #include "gen_cpp/PlanNodes_types.h" +#include "runtime/descriptors.h" #include "runtime/runtime_state.h" #include "util/runtime_profile.h" #include "util/types.h" +#include "vec/columns/column.h" #include "vec/common/string_ref.h" #include "vec/core/types.h" +#include "vec/data_types/data_type_factory.hpp" namespace doris::vectorized { VSchemaScanNode::VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, @@ -33,26 +40,13 @@ VSchemaScanNode::VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, _is_init(false), _table_name(tnode.schema_scan_node.table_name), _tuple_id(tnode.schema_scan_node.tuple_id), - _src_tuple_desc(nullptr), _dest_tuple_desc(nullptr), _tuple_idx(0), _slot_num(0), _tuple_pool(nullptr), - _schema_scanner(nullptr), - _src_tuple(nullptr), - _src_single_tuple(nullptr), - _dest_single_tuple(nullptr) {} - -VSchemaScanNode::~VSchemaScanNode() { - delete[] reinterpret_cast(_src_tuple); - _src_tuple = nullptr; + _schema_scanner(nullptr) {} - delete[] reinterpret_cast(_src_single_tuple); - _src_single_tuple = nullptr; - - delete[] reinterpret_cast(_dest_single_tuple); - _dest_single_tuple = nullptr; -} +VSchemaScanNode::~VSchemaScanNode() {} Status VSchemaScanNode::init(const TPlanNode& tnode, RuntimeState* state) { RETURN_IF_ERROR(ExecNode::init(tnode, state)); @@ -175,71 +169,42 @@ Status VSchemaScanNode::prepare(RuntimeState* state) { } RETURN_IF_ERROR(_schema_scanner->init(&_scanner_param, _pool)); - // get column info from scanner - _src_tuple_desc = _schema_scanner->tuple_desc(); - - if (nullptr == _src_tuple_desc) { - return Status::InternalError("failed to get src schema tuple desc."); - } - - _src_tuple = - reinterpret_cast(new (std::nothrow) char[_src_tuple_desc->byte_size()]); - - if (nullptr == _src_tuple) { - return Status::InternalError("new src tuple failed."); - } + const std::vector& columns_desc(_schema_scanner->get_column_desc()); - // if src tuple desc slots is zero, it's the dummy slots. - if (0 == _src_tuple_desc->slots().size()) { + // if src columns size is zero, it's the dummy slots. + if (0 == columns_desc.size()) { _slot_num = 0; } // check if type is ok. - if (_slot_num > 0) { - _index_map.resize(_slot_num); - } for (int i = 0; i < _slot_num; ++i) { // TODO(zhaochun): Is this slow? int j = 0; - for (; j < _src_tuple_desc->slots().size(); ++j) { - if (boost::iequals(_dest_tuple_desc->slots()[i]->col_name(), - _src_tuple_desc->slots()[j]->col_name())) { + for (; j < columns_desc.size(); ++j) { + if (boost::iequals(_dest_tuple_desc->slots()[i]->col_name(), columns_desc[j].name)) { break; } } - if (j >= _src_tuple_desc->slots().size()) { + if (j >= columns_desc.size()) { LOG(WARNING) << "no match column for this column(" << _dest_tuple_desc->slots()[i]->col_name() << ")"; return Status::InternalError("no match column for this column."); } - if (_src_tuple_desc->slots()[j]->type().type != _dest_tuple_desc->slots()[i]->type().type) { - LOG(WARNING) << "schema not match. input is " << _src_tuple_desc->slots()[j]->col_name() - << "(" << _src_tuple_desc->slots()[j]->type() << ") and output is " + if (columns_desc[j].type != _dest_tuple_desc->slots()[i]->type().type) { + LOG(WARNING) << "schema not match. input is " << columns_desc[j].name << "(" + << columns_desc[j].type << ") and output is " << _dest_tuple_desc->slots()[i]->col_name() << "(" << _dest_tuple_desc->slots()[i]->type() << ")"; return Status::InternalError("schema not match."); } - _index_map[i] = j; } // TODO(marcel): add int _tuple_idx indexed by TupleId somewhere in runtime_state.h _tuple_idx = 0; _is_init = true; - _src_single_tuple = - reinterpret_cast(new (std::nothrow) char[_src_tuple_desc->byte_size()]); - if (nullptr == _src_single_tuple) { - return Status::InternalError("new src single tuple failed."); - } - - _dest_single_tuple = - reinterpret_cast(new (std::nothrow) char[_dest_tuple_desc->byte_size()]); - if (nullptr == _dest_single_tuple) { - return Status::InternalError("new desc single tuple failed."); - } - return Status::OK(); } @@ -255,74 +220,57 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, return Status::InternalError("used before initialize."); } RETURN_IF_CANCELLED(state); - std::vector columns(_slot_num); bool schema_eos = false; - do { - bool mem_reuse = block->mem_reuse(); - DCHECK(block->rows() == 0); + const std::vector& columns_desc(_schema_scanner->get_column_desc()); - columns.resize(_slot_num); + do { + block->clear(); for (int i = 0; i < _slot_num; ++i) { - if (mem_reuse) { - columns[i] = std::move(*block->get_by_position(i).column).mutate(); - } else { - columns[i] = _dest_tuple_desc->slots()[i]->get_empty_mutable_column(); - } + auto dest_slot_desc = _dest_tuple_desc->slots()[i]; + block->insert(ColumnWithTypeAndName(dest_slot_desc->get_empty_mutable_column(), + dest_slot_desc->get_data_type_ptr(), + dest_slot_desc->col_name())); + } + + vectorized::Block src_block; + for (int i = 0; i < columns_desc.size(); ++i) { + TypeDescriptor descriptor(columns_desc[i].type); + auto data_type = + vectorized::DataTypeFactory::instance().create_data_type(descriptor, true); + src_block.insert(ColumnWithTypeAndName(data_type->create_column(), data_type, + columns_desc[i].name)); } while (true) { RETURN_IF_CANCELLED(state); // get all slots from schema table. - RETURN_IF_ERROR(_schema_scanner->get_next_row(_src_single_tuple, _tuple_pool.get(), - &schema_eos)); + RETURN_IF_ERROR(_schema_scanner->get_next_block(&src_block, &schema_eos)); + if (schema_eos) { *eos = true; break; } - // tuple project - project_tuple(); - for (int i = 0; i < _slot_num; ++i) { - auto slot_desc = _dest_tuple_desc->slots()[i]; - if (!slot_desc->is_materialized()) { - continue; - } - - if (_dest_single_tuple->is_null(slot_desc->null_indicator_offset())) { - if (slot_desc->is_nullable()) { - auto* nullable_column = - reinterpret_cast(columns[i].get()); - nullable_column->insert_data(nullptr, 0); - } else { - return Status::InternalError( - "nonnull column contains NULL. table={}, column={}", _table_name, - slot_desc->col_name()); - } - } else { - RETURN_IF_ERROR(write_slot_to_vectorized_column( - _dest_single_tuple->get_slot(slot_desc->tuple_offset()), slot_desc, - &columns[i])); - } - } - if (columns[0]->size() == state->batch_size()) { + if (src_block.rows() >= state->batch_size()) { break; } } - if (!columns.empty() && !columns[0]->empty()) { - auto n_columns = 0; - if (!mem_reuse) { - for (const auto slot_desc : _dest_tuple_desc->slots()) { - block->insert(ColumnWithTypeAndName(std::move(columns[n_columns++]), - slot_desc->get_data_type_ptr(), - slot_desc->col_name())); - } - } else { - columns.clear(); + + if (src_block.rows()) { + // block->check_number_of_rows(); + for (int i = 0; i < _slot_num; ++i) { + auto dest_slot_desc = _dest_tuple_desc->slots()[i]; + vectorized::MutableColumnPtr column_ptr = + std::move(*block->get_by_position(i).column).mutate(); + column_ptr->insert_range_from( + *src_block.get_by_name(dest_slot_desc->col_name()).column, 0, + src_block.rows()); } RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, _dest_tuple_desc->slots().size())); - VLOG_ROW << "VSchemaScanNode output rows: " << block->rows(); + VLOG_ROW << "VSchemaScanNode output rows: " << src_block.rows(); + src_block.clear(); } } while (block->rows() == 0 && !(*eos)); @@ -330,171 +278,6 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, return Status::OK(); } -Status VSchemaScanNode::write_slot_to_vectorized_column(void* slot, SlotDescriptor* slot_desc, - vectorized::MutableColumnPtr* column_ptr) { - vectorized::IColumn* col_ptr = column_ptr->get(); - if (slot_desc->is_nullable()) { - auto* nullable_column = reinterpret_cast(column_ptr->get()); - nullable_column->get_null_map_data().push_back(0); - col_ptr = &nullable_column->get_nested_column(); - } - switch (slot_desc->type().type) { - case TYPE_HLL: { - HyperLogLog* hll_slot = reinterpret_cast(slot); - reinterpret_cast(col_ptr)->get_data().emplace_back(*hll_slot); - break; - } - case TYPE_VARCHAR: - case TYPE_CHAR: - case TYPE_STRING: { - StringRef* str_slot = reinterpret_cast(slot); - reinterpret_cast(col_ptr)->insert_data(str_slot->data, - str_slot->size); - break; - } - - case TYPE_BOOLEAN: { - uint8_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_TINYINT: { - int8_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_SMALLINT: { - int16_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_INT: { - int32_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_BIGINT: { - int64_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_LARGEINT: { - __int128 num; - memcpy(&num, slot, sizeof(__int128)); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_FLOAT: { - float num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value( - num); - break; - } - - case TYPE_DOUBLE: { - double num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value( - num); - break; - } - - case TYPE_DATE: { - VecDateTimeValue value; - DateTimeValue* ts_slot = reinterpret_cast(slot); - value.convert_dt_to_vec_dt(ts_slot); - reinterpret_cast*>(col_ptr)->insert_data( - reinterpret_cast(&value), 0); - break; - } - - case TYPE_DATEV2: { - uint32_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_DATETIME: { - VecDateTimeValue value; - DateTimeValue* ts_slot = reinterpret_cast(slot); - value.convert_dt_to_vec_dt(ts_slot); - reinterpret_cast*>(col_ptr)->insert_data( - reinterpret_cast(&value), 0); - break; - } - - case TYPE_DATETIMEV2: { - uint32_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_DECIMALV2: { - const Int128 num = (reinterpret_cast(slot))->value; - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - case TYPE_DECIMAL128I: { - const Int128 num = (reinterpret_cast(slot))->value; - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - case TYPE_DECIMAL32: { - const int32_t num = *reinterpret_cast(slot); - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - case TYPE_DECIMAL64: { - const int64_t num = *reinterpret_cast(slot); - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - default: { - DCHECK(false) << "bad slot type: " << slot_desc->type(); - std::stringstream ss; - ss << "Fail to convert schema type:'" << slot_desc->type() << " on column:`" - << slot_desc->col_name() + "`"; - return Status::InternalError(ss.str()); - } - } - - return Status::OK(); -} - -void VSchemaScanNode::project_tuple() { - memset(_dest_single_tuple, 0, _dest_tuple_desc->num_null_bytes()); - - for (int i = 0; i < _slot_num; ++i) { - if (!_dest_tuple_desc->slots()[i]->is_materialized()) { - continue; - } - int j = _index_map[i]; - - if (_src_single_tuple->is_null(_src_tuple_desc->slots()[j]->null_indicator_offset())) { - _dest_single_tuple->set_null(_dest_tuple_desc->slots()[i]->null_indicator_offset()); - } else { - void* dest_slot = - _dest_single_tuple->get_slot(_dest_tuple_desc->slots()[i]->tuple_offset()); - void* src_slot = - _src_single_tuple->get_slot(_src_tuple_desc->slots()[j]->tuple_offset()); - int slot_size = _src_tuple_desc->slots()[j]->type().get_slot_size(); - memcpy(dest_slot, src_slot, slot_size); - } - } -} - Status VSchemaScanNode::close(RuntimeState* state) { if (is_closed()) { return Status::OK(); diff --git a/be/src/vec/exec/vschema_scan_node.h b/be/src/vec/exec/vschema_scan_node.h index 57117b23af6b79..92df6279a81b27 100644 --- a/be/src/vec/exec/vschema_scan_node.h +++ b/be/src/vec/exec/vschema_scan_node.h @@ -37,7 +37,7 @@ class VSchemaScanNode : public ScanNode { VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs); ~VSchemaScanNode(); Status prepare(RuntimeState* state) override; - virtual Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos) override; + Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos) override; // Prepare conjuncts, create Schema columns to slots mapping // initialize schema_scanner @@ -60,8 +60,6 @@ class VSchemaScanNode : public ScanNode { // Tuple id resolved in prepare() to set _tuple_desc; TupleId _tuple_id; - // Descriptor of tuples read from schema table. - const TupleDescriptor* _src_tuple_desc; // Descriptor of dest tuples const TupleDescriptor* _dest_tuple_desc; // Tuple index in tuple row. @@ -72,16 +70,6 @@ class VSchemaScanNode : public ScanNode { std::unique_ptr _tuple_pool; // Jni helper for scanning an schema table. std::unique_ptr _schema_scanner; - // Current tuple. - doris::Tuple* _src_tuple; - // Map from index in slots to column of schema table. - std::vector _index_map; - - Status write_slot_to_vectorized_column(void* slot, SlotDescriptor* slot_desc, - vectorized::MutableColumnPtr* col_ptr); - void project_tuple(); - doris::Tuple* _src_single_tuple; - doris::Tuple* _dest_single_tuple; }; } // namespace vectorized } // namespace doris \ No newline at end of file diff --git a/be/test/vec/exec/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/parquet/parquet_thrift_test.cpp index c838d11b928555..b6d635aa44a0e3 100644 --- a/be/test/vec/exec/parquet/parquet_thrift_test.cpp +++ b/be/test/vec/exec/parquet/parquet_thrift_test.cpp @@ -227,7 +227,7 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column static void create_block(std::unique_ptr& block) { // Current supported column type: - SchemaScanner::ColumnDesc column_descs[] = { + std::vector column_descs = { {"tinyint_col", TYPE_TINYINT, sizeof(int8_t), true}, {"smallint_col", TYPE_SMALLINT, sizeof(int16_t), true}, {"int_col", TYPE_INT, sizeof(int32_t), true}, @@ -246,8 +246,7 @@ static void create_block(std::unique_ptr& block) { {"date_col", TYPE_DATE, sizeof(DateTimeValue), true}, {"date_v2_col", TYPE_DATEV2, sizeof(uint32_t), true}, {"timestamp_v2_col", TYPE_DATETIMEV2, sizeof(DateTimeValue), true, 18, 0}}; - SchemaScanner schema_scanner(column_descs, - sizeof(column_descs) / sizeof(SchemaScanner::ColumnDesc)); + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool); @@ -347,7 +346,7 @@ TEST_F(ParquetThriftReaderTest, dict_decoder) { } TEST_F(ParquetThriftReaderTest, group_reader) { - SchemaScanner::ColumnDesc column_descs[] = { + std::vector column_descs = { {"tinyint_col", TYPE_TINYINT, sizeof(int8_t), true}, {"smallint_col", TYPE_SMALLINT, sizeof(int16_t), true}, {"int_col", TYPE_INT, sizeof(int32_t), true}, @@ -362,8 +361,7 @@ TEST_F(ParquetThriftReaderTest, group_reader) { {"char_col", TYPE_CHAR, sizeof(StringRef), true}, {"varchar_col", TYPE_VARCHAR, sizeof(StringRef), true}, {"date_col", TYPE_DATE, sizeof(DateTimeValue), true}}; - int num_cols = sizeof(column_descs) / sizeof(SchemaScanner::ColumnDesc); - SchemaScanner schema_scanner(column_descs, num_cols); + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool); diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp index cacc350303406d..0f1026a51a32c7 100644 --- a/be/test/vec/exprs/vexpr_test.cpp +++ b/be/test/vec/exprs/vexpr_test.cpp @@ -63,8 +63,9 @@ TEST(TEST_VEXPR, ABSTEST) { TEST(TEST_VEXPR, ABSTEST2) { using namespace doris; - SchemaScanner::ColumnDesc column_descs[] = {{"k1", TYPE_INT, sizeof(int32_t), false}}; - SchemaScanner schema_scanner(column_descs, 1); + std::vector column_descs = { + {"k1", TYPE_INT, sizeof(int32_t), false}}; + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool); diff --git a/regression-test/data/nereids_p0/system/test_query_sys_tables.out b/regression-test/data/nereids_p0/system/test_query_sys_tables.out new file mode 100644 index 00000000000000..6972d35ff6fcd3 --- /dev/null +++ b/regression-test/data/nereids_p0/system/test_query_sys_tables.out @@ -0,0 +1,46 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !backends -- +true + +-- !charsets -- +true + +-- !collations -- +true + +-- !columns -- +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 + +-- !schemata -- +internal test_query_sys_db_1 \N +internal test_query_sys_db_2 \N +internal test_query_sys_db_3 \N + +-- !tables -- +internal test_query_sys_tb_1 BASE TABLE 0 \N \N +internal test_query_sys_tb_2 BASE TABLE 0 \N \N +internal test_query_sys_tb_3 BASE TABLE 0 \N \N + +-- !session_variables -- +wait_timeout 30000 + +-- !global_variables -- +wait_timeout 31000 + +-- !user_privileges -- +'test_sys_tables'@'%' SELECT NO + +-- !views -- +test_view + diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out new file mode 100644 index 00000000000000..6972d35ff6fcd3 --- /dev/null +++ b/regression-test/data/query_p0/system/test_query_sys_tables.out @@ -0,0 +1,46 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !backends -- +true + +-- !charsets -- +true + +-- !collations -- +true + +-- !columns -- +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 + +-- !schemata -- +internal test_query_sys_db_1 \N +internal test_query_sys_db_2 \N +internal test_query_sys_db_3 \N + +-- !tables -- +internal test_query_sys_tb_1 BASE TABLE 0 \N \N +internal test_query_sys_tb_2 BASE TABLE 0 \N \N +internal test_query_sys_tb_3 BASE TABLE 0 \N \N + +-- !session_variables -- +wait_timeout 30000 + +-- !global_variables -- +wait_timeout 31000 + +-- !user_privileges -- +'test_sys_tables'@'%' SELECT NO + +-- !views -- +test_view + diff --git a/regression-test/suites/nereids_p0/system/test_query_sys_tables.groovy b/regression-test/suites/nereids_p0/system/test_query_sys_tables.groovy new file mode 100644 index 00000000000000..52bfd989e61276 --- /dev/null +++ b/regression-test/suites/nereids_p0/system/test_query_sys_tables.groovy @@ -0,0 +1,201 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_query_sys_tables", "query,p0") { + sql "SET enable_nereids_planner=true" + sql "SET enable_vectorized_engine=true" + sql "SET enable_fallback_to_original_planner=false" + def dbName1 = "test_query_sys_db_1" + def dbName2 = "test_query_sys_db_2" + def dbName3 = "test_query_sys_db_3" + def tbName1 = "test_query_sys_tb_1" + def tbName2 = "test_query_sys_tb_2" + def tbName3 = "test_query_sys_tb_3" + sql("drop database IF EXISTS ${dbName1}") + sql("drop database IF EXISTS ${dbName2}") + sql("drop database IF EXISTS ${dbName3}") + + // test backends + sql("use information_schema") + qt_backends("select count(*) >= 1 from backends") + + // test charsets + sql("use information_schema") + qt_charsets("select count(*) >= 1 from character_sets") + + // test collations + sql("use information_schema") + qt_collations("select count(*) >= 1 from collations") + + // test columns + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName2}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName3}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use information_schema") + qt_columns("select TABLE_CATALOG, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, COLUMN_TYPE, COLUMN_SIZE from columns where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'") + + // test files + // have no impl + + // test partitions + // have no impl + + // test rowsets + // have no tablet system table, add this later + + // test schemata + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + + sql("use information_schema") + qt_schemata("select CATALOG_NAME, SCHEMA_NAME, SQL_PATH from schemata where SCHEMA_NAME = '${dbName1}' or SCHEMA_NAME = '${dbName2}' or SCHEMA_NAME = '${dbName3}'"); + + // test statistics + // have no impl + + // test tables + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName2}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName3}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + + sql("use information_schema") + qt_tables("select TABLE_CATALOG, TABLE_NAME, TABLE_TYPE, AVG_ROW_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH from tables where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'"); + + // test variables + // session_variables + sql("use information_schema") + sql("SET wait_timeout = 30000") + qt_session_variables("select VARIABLE_NAME, VARIABLE_VALUE from session_variables where VARIABLE_NAME = 'wait_timeout'") + + // global_variables + sql("use information_schema") + sql("SET GLOBAL wait_timeout = 31000") + qt_global_variables("select VARIABLE_NAME, VARIABLE_VALUE from global_variables where VARIABLE_NAME = 'wait_timeout'") + + // test user_privileges + sql("CREATE USER 'test_sys_tables'") + sql("GRANT SELECT_PRIV ON *.*.* TO 'test_sys_tables'") + sql("use information_schema") + qt_user_privileges """ + select GRANTEE, PRIVILEGE_TYPE, IS_GRANTABLE from user_privileges where GRANTEE regexp '^\\'test' + """ + sql("DROP USER 'test_sys_tables'") + + // test views + sql("use ${dbName1}") + sql """ + CREATE VIEW IF NOT EXISTS ${dbName1}.test_view (a) + AS + SELECT ccc as a FROM ${tbName1} + """ + sql("use information_schema") + qt_views("select TABLE_NAME, VIEW_DEFINITION from views where TABLE_SCHEMA = '${dbName1}'") +} \ No newline at end of file diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy new file mode 100644 index 00000000000000..0565abd3823153 --- /dev/null +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_query_sys_tables", "query,p0") { + def dbName1 = "test_query_sys_db_1" + def dbName2 = "test_query_sys_db_2" + def dbName3 = "test_query_sys_db_3" + def tbName1 = "test_query_sys_tb_1" + def tbName2 = "test_query_sys_tb_2" + def tbName3 = "test_query_sys_tb_3" + sql("drop database IF EXISTS ${dbName1}") + sql("drop database IF EXISTS ${dbName2}") + sql("drop database IF EXISTS ${dbName3}") + + // test backends + sql("use information_schema") + qt_backends("select count(*) >= 1 from backends") + + // test charsets + sql("use information_schema") + qt_charsets("select count(*) >= 1 from character_sets") + + // test collations + sql("use information_schema") + qt_collations("select count(*) >= 1 from collations") + + // test columns + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName2}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName3}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use information_schema") + qt_columns("select TABLE_CATALOG, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, COLUMN_TYPE, COLUMN_SIZE from columns where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'") + + // test files + // have no impl + + // test partitions + // have no impl + + // test rowsets + // have no tablet system table, add this later + + // test schemata + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + + sql("use information_schema") + qt_schemata("select CATALOG_NAME, SCHEMA_NAME, SQL_PATH from schemata where SCHEMA_NAME = '${dbName1}' or SCHEMA_NAME = '${dbName2}' or SCHEMA_NAME = '${dbName3}'"); + + // test statistics + // have no impl + + // test tables + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName2}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName3}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + + sql("use information_schema") + qt_tables("select TABLE_CATALOG, TABLE_NAME, TABLE_TYPE, AVG_ROW_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH from tables where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'"); + + // test variables + // session_variables + sql("use information_schema") + sql("SET wait_timeout = 30000") + qt_session_variables("select VARIABLE_NAME, VARIABLE_VALUE from session_variables where VARIABLE_NAME = 'wait_timeout'") + + // global_variables + sql("use information_schema") + sql("SET GLOBAL wait_timeout = 31000") + qt_global_variables("select VARIABLE_NAME, VARIABLE_VALUE from global_variables where VARIABLE_NAME = 'wait_timeout'") + + // test user_privileges + sql("CREATE USER 'test_sys_tables'") + sql("GRANT SELECT_PRIV ON *.*.* TO 'test_sys_tables'") + sql("use information_schema") + qt_user_privileges """ + select GRANTEE, PRIVILEGE_TYPE, IS_GRANTABLE from user_privileges where GRANTEE regexp '^\\'test' + """ + sql("DROP USER 'test_sys_tables'") + + // test views + sql("use ${dbName1}") + sql """ + CREATE VIEW IF NOT EXISTS ${dbName1}.test_view (a) + AS + SELECT ccc as a FROM ${tbName1} + """ + sql("use information_schema") + qt_views("select TABLE_NAME, VIEW_DEFINITION from views where TABLE_SCHEMA = '${dbName1}'") +} \ No newline at end of file