diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index a452077f6c07e2..1d2cdf5bf2877a 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -34,33 +34,23 @@ #include "exec/schema_scanner/schema_views_scanner.h" #include "runtime/define_primitive_type.h" #include "runtime/string_value.h" +#include "vec/columns/column.h" +#include "vec/core/block.h" namespace doris { DorisServer* SchemaScanner::_s_doris_server; -SchemaScanner::SchemaScanner(ColumnDesc* columns, int column_num) +SchemaScanner::SchemaScanner(const std::vector& columns) : _is_init(false), _param(nullptr), _columns(columns), - _column_num(column_num), - _tuple_desc(nullptr), _schema_table_type(TSchemaTableType::SCH_INVALID) {} -SchemaScanner::SchemaScanner(ColumnDesc* columns, int column_num, TSchemaTableType::type type) - : _is_init(false), - _param(nullptr), - _columns(columns), - _column_num(column_num), - _tuple_desc(nullptr), - _schema_table_type(type) {} - -SchemaScanner::~SchemaScanner() { - if (_is_create_columns == true && _columns != nullptr) { - delete[] _columns; - _columns = nullptr; - } -} +SchemaScanner::SchemaScanner(const std::vector& columns, TSchemaTableType::type type) + : _is_init(false), _param(nullptr), _columns(columns), _schema_table_type(type) {} + +SchemaScanner::~SchemaScanner() = default; Status SchemaScanner::start(RuntimeState* state) { if (!_is_init) { @@ -83,6 +73,19 @@ Status SchemaScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) { return Status::OK(); } +Status SchemaScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("used before initialized."); + } + + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + *eos = true; + return Status::OK(); +} + Status SchemaScanner::init(SchemaScannerParam* param, ObjectPool* pool) { if (_is_init) { return Status::OK(); @@ -91,18 +94,22 @@ Status SchemaScanner::init(SchemaScannerParam* param, ObjectPool* pool) { return Status::InternalError("invalid parameter"); } - if (_schema_table_type == TSchemaTableType::SCH_BACKENDS) { - RETURN_IF_ERROR(create_columns(param->table_structure, pool)); - } - - if (nullptr == _columns) { + if (_columns.empty()) { return Status::InternalError("invalid parameter"); } RETURN_IF_ERROR(create_tuple_desc(pool)); + _param = param; _is_init = true; + if (_param->profile) { + _get_db_timer = ADD_TIMER(_param->profile, "GetDbTime"); + _get_table_timer = ADD_TIMER(_param->profile, "GetTableTime"); + _get_describe_timer = ADD_TIMER(_param->profile, "GetDescribeTime"); + _fill_block_timer = ADD_TIMER(_param->profile, "FillBlockTime"); + } + return Status::OK(); } @@ -145,23 +152,173 @@ SchemaScanner* SchemaScanner::create(TSchemaTableType::type type) { } } -Status SchemaScanner::create_columns(const std::vector* table_structure, - ObjectPool* pool) { - _column_num = table_structure->size(); - _columns = new ColumnDesc[_column_num]; - _is_create_columns = true; - for (size_t idx = 0; idx < table_structure->size(); ++idx) { - _columns[idx].name = table_structure->at(idx).column_name.c_str(); - _columns[idx].type = thrift_to_type(table_structure->at(idx).type); - _columns[idx].size = table_structure->at(idx).len; - _columns[idx].is_null = table_structure->at(idx).is_null; +Status SchemaScanner::fill_dest_column_for_range(vectorized::Block* block, size_t pos, + const std::vector& datas) { + const ColumnDesc& col_desc = _columns[pos]; + vectorized::MutableColumnPtr column_ptr; + column_ptr = std::move(*block->get_by_position(pos).column).assume_mutable(); + vectorized::IColumn* col_ptr = column_ptr.get(); + + auto* nullable_column = reinterpret_cast(col_ptr); + + // Resize in advance to improve insertion efficiency. + size_t fill_num = datas.size(); + col_ptr = &nullable_column->get_nested_column(); + for (int i = 0; i < fill_num; ++i) { + auto data = datas[i]; + if (data == nullptr) { + // For nested column need not insert default. + nullable_column->insert_data(nullptr, 0); + continue; + } else { + nullable_column->get_null_map_data().emplace_back(0); + } + switch (col_desc.type) { + case TYPE_HLL: { + HyperLogLog* hll_slot = reinterpret_cast(data); + reinterpret_cast(col_ptr)->get_data().emplace_back(*hll_slot); + break; + } + case TYPE_VARCHAR: + case TYPE_CHAR: + case TYPE_STRING: { + StringRef* str_slot = reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data(str_slot->data, + str_slot->size); + break; + } + + case TYPE_BOOLEAN: { + uint8_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_TINYINT: { + int8_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_SMALLINT: { + int16_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_INT: { + int32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_BIGINT: { + int64_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_LARGEINT: { + __int128 num; + memcpy(&num, data, sizeof(__int128)); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_FLOAT: { + float num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_DOUBLE: { + double num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_DATE: { + vectorized::VecDateTimeValue value; + DateTimeValue* ts_slot = reinterpret_cast(data); + value.convert_dt_to_vec_dt(ts_slot); + reinterpret_cast*>(col_ptr)->insert_data( + reinterpret_cast(&value), 0); + break; + } + + case TYPE_DATEV2: { + uint32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_DATETIME: { + vectorized::VecDateTimeValue value; + DateTimeValue* ts_slot = reinterpret_cast(data); + value.convert_dt_to_vec_dt(ts_slot); + reinterpret_cast*>(col_ptr)->insert_data( + reinterpret_cast(&value), 0); + break; + } + + case TYPE_DATETIMEV2: { + uint32_t num = *reinterpret_cast(data); + reinterpret_cast*>(col_ptr)->insert_value( + num); + break; + } + + case TYPE_DECIMALV2: { + const vectorized::Int128 num = (reinterpret_cast(data))->value; + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + case TYPE_DECIMAL128I: { + const vectorized::Int128 num = (reinterpret_cast(data))->value; + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + case TYPE_DECIMAL32: { + const int32_t num = *reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + case TYPE_DECIMAL64: { + const int64_t num = *reinterpret_cast(data); + reinterpret_cast(col_ptr)->insert_data( + reinterpret_cast(&num), 0); + break; + } + + default: { + DCHECK(false) << "bad slot type: " << col_desc.type; + std::stringstream ss; + ss << "Fail to convert schema type:'" << col_desc.type << " on column:`" + << std::string(col_desc.name) + "`"; + return Status::InternalError(ss.str()); + } + } } return Status::OK(); } Status SchemaScanner::create_tuple_desc(ObjectPool* pool) { int null_column = 0; - for (int i = 0; i < _column_num; ++i) { + for (int i = 0; i < _columns.size(); ++i) { if (_columns[i].is_null) { null_column++; } @@ -172,7 +329,7 @@ Status SchemaScanner::create_tuple_desc(ObjectPool* pool) { int null_byte = 0; int null_bit = 0; - for (int i = 0; i < _column_num; ++i) { + for (int i = 0; i < _columns.size(); ++i) { TSlotDescriptor t_slot_desc; if (_columns[i].type == TYPE_DECIMALV2) { t_slot_desc.__set_slotType(TypeDescriptor::create_decimalv2_type(27, 9).to_thrift()); diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index 9d2fa1c5fed672..a5ff0ab73b54a1 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -25,6 +25,8 @@ #include "gen_cpp/Types_types.h" #include "runtime/mem_pool.h" #include "runtime/tuple.h" +#include "util/runtime_profile.h" +#include "vec/core/block.h" namespace doris { @@ -32,6 +34,10 @@ namespace doris { class DorisServer; class RuntimeState; +namespace vectorized { +class Block; +} + // scanner parameter from frontend struct SchemaScannerParam { const std::string* db; @@ -45,6 +51,7 @@ struct SchemaScannerParam { int64_t thread_id; const std::vector* table_structure; const std::string* catalog; + std::unique_ptr profile; SchemaScannerParam() : db(nullptr), @@ -70,8 +77,8 @@ class SchemaScanner { int precision = -1; int scale = -1; }; - SchemaScanner(ColumnDesc* columns, int column_num); - SchemaScanner(ColumnDesc* columns, int column_num, TSchemaTableType::type type); + SchemaScanner(const std::vector& columns); + SchemaScanner(const std::vector& columns, TSchemaTableType::type type); virtual ~SchemaScanner(); // init object need information, schema etc. @@ -79,34 +86,36 @@ class SchemaScanner { // Start to work virtual Status start(RuntimeState* state); virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + virtual Status get_next_block(vectorized::Block* block, bool* eos); + const std::vector& get_column_desc() const { return _columns; } // factory function static SchemaScanner* create(TSchemaTableType::type type); const TupleDescriptor* tuple_desc() const { return _tuple_desc; } + const TSchemaTableType::type type() const { return _schema_table_type; } static void set_doris_server(DorisServer* doris_server) { _s_doris_server = doris_server; } protected: + Status fill_dest_column_for_range(vectorized::Block* block, size_t pos, + const std::vector& datas); Status create_tuple_desc(ObjectPool* pool); - Status create_columns(const std::vector* table_structure, - ObjectPool* pool); bool _is_init; // this is used for sub class SchemaScannerParam* _param; - // pointer to schema table's column desc - ColumnDesc* _columns; - // num of columns - int _column_num; + // schema table's column desc + std::vector _columns; TupleDescriptor* _tuple_desc; - // _is_create_columns means if ColumnDesc is created from FE. - // `_columns` should be deleted if _is_create_columns = true. - bool _is_create_columns = false; - static DorisServer* _s_doris_server; TSchemaTableType::type _schema_table_type; + + RuntimeProfile::Counter* _get_db_timer = nullptr; + RuntimeProfile::Counter* _get_table_timer = nullptr; + RuntimeProfile::Counter* _get_describe_timer = nullptr; + RuntimeProfile::Counter* _fill_block_timer = nullptr; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.cpp b/be/src/exec/schema_scanner/schema_backends_scanner.cpp index ec29eb60a64c6e..e8a606c78f774d 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_backends_scanner.cpp @@ -32,8 +32,35 @@ namespace doris { +std::vector SchemaBackendsScanner::_s_tbls_columns = { + // name, type, size + {"BackendId", TYPE_BIGINT, sizeof(StringRef), false}, + {"TabletNum", TYPE_BIGINT, sizeof(StringRef), false}, + {"HeartbeatPort", TYPE_INT, sizeof(int), false}, + {"BePort", TYPE_INT, sizeof(int), false}, + {"HttpPort", TYPE_INT, sizeof(int), false}, + {"BrpcPort", TYPE_INT, sizeof(int), false}, + {"Cluster", TYPE_VARCHAR, sizeof(StringRef), false}, + {"IP", TYPE_VARCHAR, sizeof(StringRef), false}, + {"LastStartTime", TYPE_VARCHAR, sizeof(StringRef), false}, + {"LastHeartbeat", TYPE_VARCHAR, sizeof(StringRef), false}, + {"Alive", TYPE_VARCHAR, sizeof(StringRef), false}, + {"SystemDecommissioned", TYPE_VARCHAR, sizeof(StringRef), false}, + {"ClusterDecommissioned", TYPE_VARCHAR, sizeof(StringRef), false}, + {"DataUsedCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"AvailCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"TotalCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"UsedPct", TYPE_DOUBLE, sizeof(double), false}, + {"MaxDiskUsedPct", TYPE_DOUBLE, sizeof(double), false}, + {"RemoteUsedCapacity", TYPE_BIGINT, sizeof(int64_t), false}, + {"Tag", TYPE_VARCHAR, sizeof(StringRef), false}, + {"ErrMsg", TYPE_VARCHAR, sizeof(StringRef), false}, + {"Version", TYPE_VARCHAR, sizeof(StringRef), false}, + {"Status", TYPE_VARCHAR, sizeof(StringRef), false}, +}; + SchemaBackendsScanner::SchemaBackendsScanner() - : SchemaScanner(nullptr, 0, TSchemaTableType::SCH_BACKENDS), _row_idx(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_BACKENDS), _row_idx(0) {} Status SchemaBackendsScanner::start(RuntimeState* state) { if (!_is_init) { @@ -59,9 +86,63 @@ Status SchemaBackendsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eo return _fill_one_row(tuple, pool); } +Status SchemaBackendsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + *eos = true; + return _fill_block_impl(block); +} + +Status SchemaBackendsScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto row_num = _batch_data.size(); + std::vector null_datas(row_num, nullptr); + std::vector datas(row_num); + + for (size_t col_idx = 0; col_idx < _columns.size(); ++col_idx) { + auto it = _col_name_to_type.find(_columns[col_idx].name); + if (it == _col_name_to_type.end()) { + if (_columns[col_idx].is_null) { + fill_dest_column_for_range(block, col_idx, null_datas); + } else { + return Status::InternalError( + "column {} is not found in BE, and {} is not nullable.", + _columns[col_idx].name, _columns[col_idx].name); + } + } else if (it->second == TYPE_BIGINT) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + datas[row_idx] = &_batch_data[row_idx].column_value[col_idx].longVal; + } + fill_dest_column_for_range(block, col_idx, datas); + } else if (it->second == TYPE_INT) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + datas[row_idx] = &_batch_data[row_idx].column_value[col_idx].intVal; + } + fill_dest_column_for_range(block, col_idx, datas); + } else if (it->second == TYPE_VARCHAR) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + datas[row_idx] = &_batch_data[row_idx].column_value[col_idx].stringVal; + } + fill_dest_column_for_range(block, col_idx, datas); + } else if (it->second == TYPE_DOUBLE) { + for (int row_idx = 0; row_idx < row_num; ++row_idx) { + datas[row_idx] = &_batch_data[row_idx].column_value[col_idx].doubleVal; + } + fill_dest_column_for_range(block, col_idx, datas); + } else { + // other type + } + } + return Status::OK(); +} + Status SchemaBackendsScanner::_fill_one_row(Tuple* tuple, MemPool* pool) { memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - for (size_t col_idx = 0; col_idx < _column_num; ++col_idx) { + for (size_t col_idx = 0; col_idx < _columns.size(); ++col_idx) { RETURN_IF_ERROR(_fill_one_col(tuple, pool, col_idx)); } ++_row_idx; diff --git a/be/src/exec/schema_scanner/schema_backends_scanner.h b/be/src/exec/schema_scanner/schema_backends_scanner.h index 32753f568e5b18..8995b029ddbed5 100644 --- a/be/src/exec/schema_scanner/schema_backends_scanner.h +++ b/be/src/exec/schema_scanner/schema_backends_scanner.h @@ -27,8 +27,10 @@ class SchemaBackendsScanner : public SchemaScanner { Status start(RuntimeState* state) override; Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: + Status _fill_block_impl(vectorized::Block* block); Status _fill_one_row(Tuple* tuple, MemPool* pool); Status _fetch_backends_info(); Status _fill_one_col(Tuple* tuple, MemPool* pool, size_t idx); @@ -37,6 +39,7 @@ class SchemaBackendsScanner : public SchemaScanner { private: // column_name -> type, set by _set_col_name_to_type() std::unordered_map _col_name_to_type; + static std::vector _s_tbls_columns; std::vector _batch_data; size_t _row_idx; diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp index 27772c85bfe436..e029d65ab7a582 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp @@ -21,7 +21,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaCharsetsScanner::_s_css_columns[] = { +std::vector SchemaCharsetsScanner::_s_css_columns = { // name, type, size {"CHARACTER_SET_NAME", TYPE_VARCHAR, sizeof(StringValue), false}, {"DEFAULT_COLLATE_NAME", TYPE_VARCHAR, sizeof(StringValue), false}, @@ -35,8 +35,7 @@ SchemaCharsetsScanner::CharsetStruct SchemaCharsetsScanner::_s_charsets[] = { }; SchemaCharsetsScanner::SchemaCharsetsScanner() - : SchemaScanner(_s_css_columns, sizeof(_s_css_columns) / sizeof(SchemaScanner::ColumnDesc)), - _index(0) {} + : SchemaScanner(_s_css_columns, TSchemaTableType::SCH_CHARSETS), _index(0) {} SchemaCharsetsScanner::~SchemaCharsetsScanner() {} @@ -101,4 +100,63 @@ Status SchemaCharsetsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eo return fill_one_row(tuple, pool); } +Status SchemaCharsetsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("call this before initial."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("invalid parameter."); + } + *eos = true; + return _fill_block_impl(block); +} + +Status SchemaCharsetsScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto row_num = 0; + while (nullptr != _s_charsets[row_num].charset) { + ++row_num; + } + std::vector datas(row_num); + + // variables names + { + StringRef strs[row_num]; + for (int i = 0; i < row_num; ++i) { + strs[i] = StringRef(_s_charsets[i].charset, strlen(_s_charsets[i].charset)); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 0, datas); + } + // DEFAULT_COLLATE_NAME + { + StringRef strs[row_num]; + for (int i = 0; i < row_num; ++i) { + strs[i] = StringRef(_s_charsets[i].default_collation, + strlen(_s_charsets[i].default_collation)); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 1, datas); + } + // DESCRIPTION + { + StringRef strs[row_num]; + for (int i = 0; i < row_num; ++i) { + strs[i] = StringRef(_s_charsets[i].description, strlen(_s_charsets[i].description)); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 2, datas); + } + // maxlen + { + int64_t srcs[row_num]; + for (int i = 0; i < row_num; ++i) { + srcs[i] = _s_charsets[i].maxlen; + datas[i] = srcs + i; + } + fill_dest_column_for_range(block, 3, datas); + } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.h b/be/src/exec/schema_scanner/schema_charsets_scanner.h index 3a9f07100ae342..92fde478c64063 100644 --- a/be/src/exec/schema_scanner/schema_charsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_charsets_scanner.h @@ -26,9 +26,10 @@ namespace doris { class SchemaCharsetsScanner : public SchemaScanner { public: SchemaCharsetsScanner(); - virtual ~SchemaCharsetsScanner(); + ~SchemaCharsetsScanner() override; - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: struct CharsetStruct { @@ -39,9 +40,10 @@ class SchemaCharsetsScanner : public SchemaScanner { }; Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); int _index; - static SchemaScanner::ColumnDesc _s_css_columns[]; + static std::vector _s_css_columns; static CharsetStruct _s_charsets[]; }; diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.cpp b/be/src/exec/schema_scanner/schema_collations_scanner.cpp index 304eec82b1d587..a292e540f1cad7 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_collations_scanner.cpp @@ -22,7 +22,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaCollationsScanner::_s_cols_columns[] = { +std::vector SchemaCollationsScanner::_s_cols_columns = { // name, type, size {"COLLATION_NAME", TYPE_VARCHAR, sizeof(StringValue), false}, {"CHARACTER_SET_NAME", TYPE_VARCHAR, sizeof(StringValue), false}, @@ -38,9 +38,7 @@ SchemaCollationsScanner::CollationStruct SchemaCollationsScanner::_s_collations[ }; SchemaCollationsScanner::SchemaCollationsScanner() - : SchemaScanner(_s_cols_columns, - sizeof(_s_cols_columns) / sizeof(SchemaScanner::ColumnDesc)), - _index(0) {} + : SchemaScanner(_s_cols_columns, TSchemaTableType::SCH_COLLATIONS), _index(0) {} SchemaCollationsScanner::~SchemaCollationsScanner() {} @@ -123,4 +121,81 @@ Status SchemaCollationsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* return fill_one_row(tuple, pool); } +Status SchemaCollationsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("call this before initial."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("invalid parameter."); + } + + *eos = true; + return _fill_block_impl(block); +} + +Status SchemaCollationsScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto row_num = 0; + while (nullptr != _s_collations[row_num].name) { + ++row_num; + } + std::vector datas(row_num); + + // COLLATION_NAME + { + StringRef strs[row_num]; + for (int i = 0; i < row_num; ++i) { + strs[i] = StringRef(_s_collations[i].name, strlen(_s_collations[i].name)); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 0, datas); + } + // charset + { + StringRef strs[row_num]; + for (int i = 0; i < row_num; ++i) { + strs[i] = StringRef(_s_collations[i].charset, strlen(_s_collations[i].charset)); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 1, datas); + } + // id + { + int64_t srcs[row_num]; + for (int i = 0; i < row_num; ++i) { + srcs[i] = _s_collations[i].id; + datas[i] = srcs + i; + } + fill_dest_column_for_range(block, 2, datas); + } + // is_default + { + StringRef strs[row_num]; + for (int i = 0; i < row_num; ++i) { + strs[i] = StringRef(_s_collations[i].is_default, strlen(_s_collations[i].is_default)); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 3, datas); + } + // IS_COMPILED + { + StringRef strs[row_num]; + for (int i = 0; i < row_num; ++i) { + strs[i] = StringRef(_s_collations[i].is_compile, strlen(_s_collations[i].is_compile)); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 4, datas); + } + // sortlen + { + int64_t srcs[row_num]; + for (int i = 0; i < row_num; ++i) { + srcs[i] = _s_collations[i].sortlen; + datas[i] = srcs + i; + } + fill_dest_column_for_range(block, 5, datas); + } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.h b/be/src/exec/schema_scanner/schema_collations_scanner.h index 66e3faeed33e69..17c08567fd0171 100644 --- a/be/src/exec/schema_scanner/schema_collations_scanner.h +++ b/be/src/exec/schema_scanner/schema_collations_scanner.h @@ -26,9 +26,10 @@ namespace doris { class SchemaCollationsScanner : public SchemaScanner { public: SchemaCollationsScanner(); - virtual ~SchemaCollationsScanner(); + ~SchemaCollationsScanner() override; - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: struct CollationStruct { @@ -41,9 +42,10 @@ class SchemaCollationsScanner : public SchemaScanner { }; Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); int _index; - static SchemaScanner::ColumnDesc _s_cols_columns[]; + static std::vector _s_cols_columns; static CollationStruct _s_collations[]; }; diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp index c471a5f98afc5a..963ed40a672eea 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp @@ -25,7 +25,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaColumnsScanner::_s_col_columns[] = { +std::vector SchemaColumnsScanner::_s_col_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringValue), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringValue), false}, @@ -54,7 +54,7 @@ SchemaScanner::ColumnDesc SchemaColumnsScanner::_s_col_columns[] = { }; SchemaColumnsScanner::SchemaColumnsScanner() - : SchemaScanner(_s_col_columns, sizeof(_s_col_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_col_columns, TSchemaTableType::SCH_COLUMNS), _db_index(0), _table_index(0), _column_index(0) {} @@ -62,6 +62,7 @@ SchemaColumnsScanner::SchemaColumnsScanner() SchemaColumnsScanner::~SchemaColumnsScanner() {} Status SchemaColumnsScanner::start(RuntimeState* state) { + SCOPED_TIMER(_get_db_timer); if (!_is_init) { return Status::InternalError("schema columns scanner not inited."); } @@ -95,7 +96,7 @@ Status SchemaColumnsScanner::start(RuntimeState* state) { } //For compatibility with mysql the result of DATA_TYPE in information_schema.columns -std::string SchemaColumnsScanner::to_mysql_data_type_string(TColumnDesc& desc) { +std::string SchemaColumnsScanner::_to_mysql_data_type_string(TColumnDesc& desc) { switch (desc.columnType) { case TPrimitiveType::BOOLEAN: return "tinyint"; @@ -144,7 +145,7 @@ std::string SchemaColumnsScanner::to_mysql_data_type_string(TColumnDesc& desc) { } } -std::string SchemaColumnsScanner::type_to_string(TColumnDesc& desc) { +std::string SchemaColumnsScanner::_type_to_string(TColumnDesc& desc) { switch (desc.columnType) { case TPrimitiveType::BOOLEAN: return "tinyint(1)"; @@ -231,7 +232,7 @@ std::string SchemaColumnsScanner::type_to_string(TColumnDesc& desc) { //fill row in the "INFORMATION_SCHEMA COLUMNS" //Reference from https://dev.mysql.com/doc/refman/8.0/en/information-schema-columns-table.html -Status SchemaColumnsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { +Status SchemaColumnsScanner::_fill_one_row(Tuple* tuple, MemPool* pool) { // set all bit to not null memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); @@ -309,7 +310,7 @@ Status SchemaColumnsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { void* slot = tuple->get_slot(_tuple_desc->slots()[7]->tuple_offset()); StringValue* str_slot = reinterpret_cast(slot); std::string buffer = - to_mysql_data_type_string(_desc_result.columns[_column_index].columnDesc); + _to_mysql_data_type_string(_desc_result.columns[_column_index].columnDesc); str_slot->len = buffer.length(); str_slot->ptr = (char*)pool->allocate(str_slot->len); memcpy(str_slot->ptr, buffer.c_str(), str_slot->len); @@ -378,7 +379,7 @@ Status SchemaColumnsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { { void* slot = tuple->get_slot(_tuple_desc->slots()[15]->tuple_offset()); StringValue* str_slot = reinterpret_cast(slot); - std::string buffer = type_to_string(_desc_result.columns[_column_index].columnDesc); + std::string buffer = _type_to_string(_desc_result.columns[_column_index].columnDesc); str_slot->len = buffer.length(); str_slot->ptr = (char*)pool->allocate(str_slot->len); memcpy(str_slot->ptr, buffer.c_str(), str_slot->len); @@ -451,7 +452,8 @@ Status SchemaColumnsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { return Status::OK(); } -Status SchemaColumnsScanner::get_new_desc() { +Status SchemaColumnsScanner::_get_new_desc() { + SCOPED_TIMER(_get_describe_timer); TDescribeTableParams desc_params; desc_params.__set_db(_db_result.dbs[_db_index - 1]); if (_db_result.__isset.catalogs) { @@ -480,7 +482,46 @@ Status SchemaColumnsScanner::get_new_desc() { return Status::OK(); } -Status SchemaColumnsScanner::get_new_table() { +Status SchemaColumnsScanner::_get_new_descs() { + SCOPED_TIMER(_get_describe_timer); + TDescribeTablesParams desc_params; + desc_params.__set_db(_db_result.dbs[_db_index - 1]); + if (_db_result.__isset.catalogs) { + desc_params.__set_catalog(_db_result.catalogs[_db_index - 1]); + } + for (int i = 0; i < 100; ++i) { + if (_table_index >= _table_result.tables.size()) { + break; + } + desc_params.tables_name.push_back(_table_result.tables[_table_index++]); + } + LOG(WARNING) << "_get_new_desc tables_name size: " << desc_params.tables_name.size(); + if (nullptr != _param->current_user_ident) { + desc_params.__set_current_user_ident(*(_param->current_user_ident)); + } else { + if (nullptr != _param->user) { + desc_params.__set_user(*(_param->user)); + } + if (nullptr != _param->user_ip) { + desc_params.__set_user_ip(*(_param->user_ip)); + } + } + + if (nullptr != _param->ip && 0 != _param->port) { + RETURN_IF_ERROR(SchemaHelper::describe_tables(*(_param->ip), _param->port, desc_params, + &_descs_result)); + } else { + return Status::InternalError("IP or port doesn't exists"); + } + + // compatible _fill_one_row() + _column_index = 0; + + return Status::OK(); +} + +Status SchemaColumnsScanner::_get_new_table() { + SCOPED_TIMER(_get_table_timer); TGetTablesParams table_params; table_params.__set_db(_db_result.dbs[_db_index]); if (_db_result.__isset.catalogs) { @@ -521,18 +562,292 @@ Status SchemaColumnsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos while (_column_index >= _desc_result.columns.size()) { if (_table_index >= _table_result.tables.size()) { if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); } else { *eos = true; return Status::OK(); } } else { - RETURN_IF_ERROR(get_new_desc()); + RETURN_IF_ERROR(_get_new_desc()); } } *eos = false; - return fill_one_row(tuple, pool); + return _fill_one_row(tuple, pool); +} + +Status SchemaColumnsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("use this class before inited."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input parameter is nullptr."); + } + + while (_table_index >= _table_result.tables.size()) { + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(_get_new_table()); + } else { + *eos = true; + return Status::OK(); + } + } + RETURN_IF_ERROR(_get_new_descs()); + + *eos = false; + return _fill_block_impl(block); +} + +Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto columns_num = _descs_result.columns.size(); + std::vector null_datas(columns_num, nullptr); + std::vector datas(columns_num); + // TABLE_CATALOG + { + if (!_db_result.__isset.catalogs) { + fill_dest_column_for_range(block, 0, null_datas); + } else { + std::string catalog_name = _db_result.catalogs[_db_index - 1]; + StringRef str = StringRef(catalog_name.c_str(), catalog_name.size()); + for (int i = 0; i < columns_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 0, datas); + } + } + // TABLE_SCHEMA + { + std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); + StringRef str = StringRef(db_name.c_str(), db_name.size()); + for (int i = 0; i < columns_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 1, datas); + } + // TABLE_NAME + { + StringRef strs[columns_num]; + int offset_index = 0; + int cur_table_index = _table_index - _descs_result.tables_offset.size(); + + for (int i = 0; i < columns_num; ++i) { + while (_descs_result.tables_offset[offset_index] <= i) { + ++offset_index; + ++cur_table_index; + } + strs[i] = StringRef(_table_result.tables[cur_table_index].c_str(), + _table_result.tables[cur_table_index].length()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 2, datas); + } + // COLUMN_NAME + { + StringRef strs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + strs[i] = StringRef(_descs_result.columns[i].columnDesc.columnName.c_str(), + _descs_result.columns[i].columnDesc.columnName.length()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 3, datas); + } + // ORDINAL_POSITION + { + int64_t srcs[columns_num]; + int offset_index = 0; + int columns_index = 1; + for (int i = 0; i < columns_num; ++i) { + while (_descs_result.tables_offset[offset_index] <= i) { + ++offset_index; + columns_index = 1; + } + srcs[i] = columns_index++; + datas[i] = srcs + i; + } + fill_dest_column_for_range(block, 4, datas); + } + // COLUMN_DEFAULT + { fill_dest_column_for_range(block, 5, null_datas); } + // IS_NULLABLE + { + StringRef str_yes = StringRef("YES", 3); + StringRef str_no = StringRef("NO", 2); + for (int i = 0; i < columns_num; ++i) { + if (_descs_result.columns[i].columnDesc.__isset.isAllowNull) { + if (_descs_result.columns[i].columnDesc.isAllowNull) { + datas[i] = &str_yes; + } else { + datas[i] = &str_no; + } + } else { + datas[i] = &str_no; + } + } + fill_dest_column_for_range(block, 6, datas); + } + // DATA_TYPE + { + std::string buffers[columns_num]; + StringRef strs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + buffers[i] = _to_mysql_data_type_string(_descs_result.columns[i].columnDesc); + strs[i] = StringRef(buffers[i].c_str(), buffers[i].length()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 7, datas); + } + // CHARACTER_MAXIMUM_LENGTH + // For string columns, the maximum length in characters. + { + int64_t srcs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + int data_type = _descs_result.columns[i].columnDesc.columnType; + if (data_type == TPrimitiveType::VARCHAR || data_type == TPrimitiveType::CHAR || + data_type == TPrimitiveType::STRING) { + if (_descs_result.columns[i].columnDesc.__isset.columnLength) { + srcs[i] = _descs_result.columns[i].columnDesc.columnLength; + datas[i] = srcs + i; + } else { + datas[i] = nullptr; + } + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 8, datas); + } + // CHARACTER_OCTET_LENGTH + // For string columns, the maximum length in bytes. + { + int64_t srcs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + int data_type = _descs_result.columns[i].columnDesc.columnType; + if (data_type == TPrimitiveType::VARCHAR || data_type == TPrimitiveType::CHAR || + data_type == TPrimitiveType::STRING) { + if (_descs_result.columns[i].columnDesc.__isset.columnLength) { + srcs[i] = _descs_result.columns[i].columnDesc.columnLength * 4; + datas[i] = srcs + i; + } else { + datas[i] = nullptr; + } + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 9, datas); + } + // NUMERIC_PRECISION + { + int64_t srcs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + if (_descs_result.columns[i].columnDesc.__isset.columnPrecision) { + srcs[i] = _descs_result.columns[i].columnDesc.columnPrecision; + datas[i] = srcs + i; + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 10, datas); + } + // NUMERIC_SCALE + { + int64_t srcs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + if (_descs_result.columns[i].columnDesc.__isset.columnScale) { + srcs[i] = _descs_result.columns[i].columnDesc.columnScale; + datas[i] = srcs + i; + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 11, datas); + } + // DATETIME_PRECISION + { fill_dest_column_for_range(block, 12, null_datas); } + // CHARACTER_SET_NAME + { fill_dest_column_for_range(block, 13, null_datas); } + // COLLATION_NAME + { fill_dest_column_for_range(block, 14, null_datas); } + // COLUMN_TYPE + { + std::string buffers[columns_num]; + StringRef strs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + buffers[i] = _type_to_string(_descs_result.columns[i].columnDesc); + strs[i] = StringRef(buffers[i].c_str(), buffers[i].length()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 15, datas); + } + // COLUMN_KEY + { + StringRef str = StringRef("", 0); + StringRef strs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + if (_descs_result.columns[i].columnDesc.__isset.columnKey) { + strs[i] = StringRef(_descs_result.columns[i].columnDesc.columnKey.c_str(), + _descs_result.columns[i].columnDesc.columnKey.length()); + datas[i] = strs + i; + } else { + datas[i] = &str; + } + } + fill_dest_column_for_range(block, 16, datas); + } + // EXTRA + { + StringRef str = StringRef("", 0); + std::vector datas(columns_num, &str); + fill_dest_column_for_range(block, 17, datas); + } + // PRIVILEGES + { + StringRef str = StringRef("", 0); + std::vector datas(columns_num, &str); + fill_dest_column_for_range(block, 18, datas); + } + // COLUMN_COMMENT + { + StringRef strs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + strs[i] = StringRef(_descs_result.columns[i].comment.c_str(), + _descs_result.columns[i].comment.length()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 19, datas); + } + // COLUMN_SIZE + { + int64_t srcs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + if (_descs_result.columns[i].columnDesc.__isset.columnLength) { + srcs[i] = _descs_result.columns[i].columnDesc.columnLength; + datas[i] = srcs + i; + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 20, datas); + } + // DECIMAL_DIGITS + { + int64_t srcs[columns_num]; + for (int i = 0; i < columns_num; ++i) { + if (_descs_result.columns[i].columnDesc.__isset.columnScale) { + srcs[i] = _descs_result.columns[i].columnDesc.columnScale; + datas[i] = srcs + i; + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 21, datas); + } + // GENERATION_EXPRESSION + { fill_dest_column_for_range(block, 22, null_datas); } + // SRS_ID + { fill_dest_column_for_range(block, 23, null_datas); } + return Status::OK(); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.h b/be/src/exec/schema_scanner/schema_columns_scanner.h index 25afa984d6ed34..180ff847121224 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.h +++ b/be/src/exec/schema_scanner/schema_columns_scanner.h @@ -27,17 +27,20 @@ namespace doris { class SchemaColumnsScanner : public SchemaScanner { public: SchemaColumnsScanner(); - virtual ~SchemaColumnsScanner(); - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + ~SchemaColumnsScanner() override; + Status start(RuntimeState* state) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); - Status fill_one_row(Tuple* tuple, MemPool* pool); - Status get_new_desc(); - Status get_create_table(std::string* result); - std::string to_mysql_data_type_string(TColumnDesc& desc); - std::string type_to_string(TColumnDesc& desc); + Status _get_new_table(); + Status _fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); + Status _get_new_desc(); + Status _get_new_descs(); + Status _get_create_table(std::string* result); + std::string _to_mysql_data_type_string(TColumnDesc& desc); + std::string _type_to_string(TColumnDesc& desc); int _db_index; int _table_index; @@ -45,7 +48,8 @@ class SchemaColumnsScanner : public SchemaScanner { TGetDbsResult _db_result; TGetTablesResult _table_result; TDescribeTableResult _desc_result; - static SchemaScanner::ColumnDesc _s_col_columns[]; + TDescribeTablesResult _descs_result; + static std::vector _s_col_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp index 665947fdd839ef..539b5d379113e0 100644 --- a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp @@ -18,12 +18,12 @@ #include "schema_dummy_scanner.h" namespace { -doris::SchemaScanner::ColumnDesc DUMMY_COLUMN; +std::vector DUMMY_COLUMN; } namespace doris { -SchemaDummyScanner::SchemaDummyScanner() : SchemaScanner(&DUMMY_COLUMN, 0) {} +SchemaDummyScanner::SchemaDummyScanner() : SchemaScanner(DUMMY_COLUMN) {} SchemaDummyScanner::~SchemaDummyScanner() {} @@ -36,4 +36,9 @@ Status SchemaDummyScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) return Status::OK(); } +Status SchemaDummyScanner::get_next_block(vectorized::Block* block, bool* eos) { + *eos = true; + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.h b/be/src/exec/schema_scanner/schema_dummy_scanner.h index 2b661ae58bad24..66775babbe5e49 100644 --- a/be/src/exec/schema_scanner/schema_dummy_scanner.h +++ b/be/src/exec/schema_scanner/schema_dummy_scanner.h @@ -24,9 +24,10 @@ namespace doris { class SchemaDummyScanner : public SchemaScanner { public: SchemaDummyScanner(); - virtual ~SchemaDummyScanner(); - virtual Status start(RuntimeState* state = nullptr); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + ~SchemaDummyScanner() override; + Status start(RuntimeState* state = nullptr) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_files_scanner.cpp b/be/src/exec/schema_scanner/schema_files_scanner.cpp index d237d3da6edebb..dc96f080bc2e51 100644 --- a/be/src/exec/schema_scanner/schema_files_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_files_scanner.cpp @@ -20,10 +20,11 @@ #include "exec/schema_scanner/schema_helper.h" #include "runtime/primitive_type.h" #include "runtime/string_value.h" +#include "util/runtime_profile.h" namespace doris { -SchemaScanner::ColumnDesc SchemaFilesScanner::_s_tbls_columns[] = { +std::vector SchemaFilesScanner::_s_tbls_columns = { // name, type, size, is_null {"FILE_ID", TYPE_BIGINT, sizeof(int64_t), true}, {"FILE_NAME", TYPE_STRING, sizeof(StringValue), true}, @@ -66,8 +67,7 @@ SchemaScanner::ColumnDesc SchemaFilesScanner::_s_tbls_columns[] = { }; SchemaFilesScanner::SchemaFilesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_FILES), _db_index(0), _table_index(0) {} @@ -77,25 +77,26 @@ Status SchemaFilesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } + SCOPED_TIMER(_get_db_timer); TGetDbsParams db_params; - if (NULL != _param->db) { + if (nullptr != _param->db) { db_params.__set_pattern(*(_param->db)); } if (nullptr != _param->catalog) { db_params.__set_catalog(*(_param->catalog)); } - if (NULL != _param->current_user_ident) { + if (nullptr != _param->current_user_ident) { db_params.__set_current_user_ident(*(_param->current_user_ident)); } else { - if (NULL != _param->user) { + if (nullptr != _param->user) { db_params.__set_user(*(_param->user)); } - if (NULL != _param->user_ip) { + if (nullptr != _param->user_ip) { db_params.__set_user_ip(*(_param->user_ip)); } } - if (NULL != _param->ip && 0 != _param->port) { + if (nullptr != _param->ip && 0 != _param->port) { RETURN_IF_ERROR( SchemaHelper::get_db_names(*(_param->ip), _param->port, db_params, &_db_result)); } else { @@ -115,4 +116,15 @@ Status SchemaFilesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) return Status::OK(); } +Status SchemaFilesScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + *eos = true; + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_files_scanner.h b/be/src/exec/schema_scanner/schema_files_scanner.h index 012d1ed097c6a1..1008ff4917cb07 100644 --- a/be/src/exec/schema_scanner/schema_files_scanner.h +++ b/be/src/exec/schema_scanner/schema_files_scanner.h @@ -25,16 +25,17 @@ namespace doris { class SchemaFilesScanner : public SchemaScanner { public: SchemaFilesScanner(); - virtual ~SchemaFilesScanner(); + ~SchemaFilesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; int _db_index; int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_helper.cpp b/be/src/exec/schema_scanner/schema_helper.cpp index d31edd0da304a2..2cead73aef34ea 100644 --- a/be/src/exec/schema_scanner/schema_helper.cpp +++ b/be/src/exec/schema_scanner/schema_helper.cpp @@ -61,6 +61,49 @@ Status SchemaHelper::describe_table(const std::string& ip, const int32_t port, }); } +Status SchemaHelper::describe_tables(const std::string& ip, const int32_t port, + const TDescribeTablesParams& request, + TDescribeTablesResult* result) { + Status rpcStatus = ThriftRpcHelper::rpc( + ip, port, [&request, &result](FrontendServiceConnection& client) { + client->describeTables(*result, request); + }); + // FE have no describeTables rpc service + if (!rpcStatus.ok()) { + TDescribeTableParams single_table_request; + single_table_request.__set_db(request.db); + if (request.__isset.catalog) { + single_table_request.__set_catalog(request.catalog); + } + + if (request.__isset.current_user_ident) { + single_table_request.__set_current_user_ident(request.current_user_ident); + } else { + if (request.__isset.user) { + single_table_request.__set_user(request.user); + } + if (request.__isset.user_ip) { + single_table_request.__set_user_ip(request.user_ip); + } + } + const auto& tables = request.tables_name; + result->columns.clear(); + result->tables_offset.clear(); + for (int i = 0; i < tables.size(); ++i) { + single_table_request.__set_table_name(tables[i]); + TDescribeTableResult single_table_result; + RETURN_IF_ERROR(describe_table(ip, port, single_table_request, &single_table_result)); + const auto& columns = single_table_result.columns; + for (int j = 0; j < columns.size(); ++j) { + result->columns.emplace_back(columns[j]); + } + result->tables_offset.emplace_back(result->columns.size()); + } + } + + return rpcStatus; +} + Status SchemaHelper::show_variables(const std::string& ip, const int32_t port, const TShowVariableRequest& request, TShowVariableResult* result) { diff --git a/be/src/exec/schema_scanner/schema_helper.h b/be/src/exec/schema_scanner/schema_helper.h index 12c17ee62513c3..a6a2ed976e889f 100644 --- a/be/src/exec/schema_scanner/schema_helper.h +++ b/be/src/exec/schema_scanner/schema_helper.h @@ -40,6 +40,10 @@ class SchemaHelper { const TDescribeTableParams& desc_params, TDescribeTableResult* desc_result); + static Status describe_tables(const std::string& ip, const int32_t port, + const TDescribeTablesParams& desc_params, + TDescribeTablesResult* desc_result); + static Status show_variables(const std::string& ip, const int32_t port, const TShowVariableRequest& var_params, TShowVariableResult* var_result); diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp index 018e86ede5849d..37928b1e391fb5 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp @@ -21,10 +21,11 @@ #include "runtime/datetime_value.h" #include "runtime/primitive_type.h" #include "runtime/string_value.h" +#include "util/runtime_profile.h" namespace doris { -SchemaScanner::ColumnDesc SchemaPartitionsScanner::_s_tbls_columns[] = { +std::vector SchemaPartitionsScanner::_s_tbls_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringValue), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringValue), true}, @@ -54,8 +55,7 @@ SchemaScanner::ColumnDesc SchemaPartitionsScanner::_s_tbls_columns[] = { }; SchemaPartitionsScanner::SchemaPartitionsScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_PARTITIONS), _db_index(0), _table_index(0) {} @@ -65,25 +65,26 @@ Status SchemaPartitionsScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } + SCOPED_TIMER(_get_db_timer); TGetDbsParams db_params; - if (NULL != _param->db) { + if (nullptr != _param->db) { db_params.__set_pattern(*(_param->db)); } if (nullptr != _param->catalog) { db_params.__set_catalog(*(_param->catalog)); } - if (NULL != _param->current_user_ident) { + if (nullptr != _param->current_user_ident) { db_params.__set_current_user_ident(*(_param->current_user_ident)); } else { - if (NULL != _param->user) { + if (nullptr != _param->user) { db_params.__set_user(*(_param->user)); } - if (NULL != _param->user_ip) { + if (nullptr != _param->user_ip) { db_params.__set_user_ip(*(_param->user_ip)); } } - if (NULL != _param->ip && 0 != _param->port) { + if (nullptr != _param->ip && 0 != _param->port) { RETURN_IF_ERROR( SchemaHelper::get_db_names(*(_param->ip), _param->port, db_params, &_db_result)); } else { @@ -103,4 +104,15 @@ Status SchemaPartitionsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* return Status::OK(); } +Status SchemaPartitionsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + *eos = true; + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.h b/be/src/exec/schema_scanner/schema_partitions_scanner.h index 5bce10cfbbaa17..766844568bfb91 100644 --- a/be/src/exec/schema_scanner/schema_partitions_scanner.h +++ b/be/src/exec/schema_scanner/schema_partitions_scanner.h @@ -25,16 +25,17 @@ namespace doris { class SchemaPartitionsScanner : public SchemaScanner { public: SchemaPartitionsScanner(); - virtual ~SchemaPartitionsScanner(); + ~SchemaPartitionsScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; int _db_index; int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp index 00314b83062f74..582e44a4e61cf9 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp @@ -31,7 +31,7 @@ #include "runtime/primitive_type.h" #include "runtime/string_value.h" namespace doris { -SchemaScanner::ColumnDesc SchemaRowsetsScanner::_s_tbls_columns[] = { +std::vector SchemaRowsetsScanner::_s_tbls_columns = { // name, type, size, is_null {"BACKEND_ID", TYPE_BIGINT, sizeof(int64_t), true}, {"ROWSET_ID", TYPE_VARCHAR, sizeof(StringValue), true}, @@ -50,17 +50,16 @@ SchemaScanner::ColumnDesc SchemaRowsetsScanner::_s_tbls_columns[] = { }; SchemaRowsetsScanner::SchemaRowsetsScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_ROWSETS), backend_id_(0), - rowsets_idx_(0) {}; + _rowsets_idx(0) {}; Status SchemaRowsetsScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } backend_id_ = state->backend_id(); - RETURN_IF_ERROR(get_all_rowsets()); + RETURN_IF_ERROR(_get_all_rowsets()); return Status::OK(); } @@ -71,7 +70,7 @@ Status SchemaRowsetsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos if (nullptr == tuple || nullptr == pool || nullptr == eos) { return Status::InternalError("input pointer is nullptr."); } - if (rowsets_idx_ >= rowsets_.size()) { + if (_rowsets_idx >= rowsets_.size()) { *eos = true; return Status::OK(); } @@ -79,7 +78,7 @@ Status SchemaRowsetsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos return fill_one_row(tuple, pool); } -Status SchemaRowsetsScanner::get_all_rowsets() { +Status SchemaRowsetsScanner::_get_all_rowsets() { std::vector tablets = StorageEngine::instance()->tablet_manager()->get_all_tablet(); for (const auto& tablet : tablets) { @@ -100,7 +99,7 @@ Status SchemaRowsetsScanner::get_all_rowsets() { Status SchemaRowsetsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { // set all bit to not null memset((void*)tuple, 0, _tuple_desc->num_null_bytes()); - RowsetSharedPtr rowset = rowsets_[rowsets_idx_]; + RowsetSharedPtr rowset = rowsets_[_rowsets_idx]; // BACKEND_ID { void* slot = tuple->get_slot(_tuple_desc->slots()[0]->tuple_offset()); @@ -170,7 +169,154 @@ Status SchemaRowsetsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { void* slot = tuple->get_slot(_tuple_desc->slots()[12]->tuple_offset()); *(reinterpret_cast(slot)) = rowset->newest_write_timestamp(); } - ++rowsets_idx_; + ++_rowsets_idx; + return Status::OK(); +} + +Status SchemaRowsetsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + if (_rowsets_idx >= rowsets_.size()) { + *eos = true; + return Status::OK(); + } + *eos = false; + return _fill_block_impl(block); +} + +Status SchemaRowsetsScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + size_t fill_rowsets_num = std::min(1000ul, rowsets_.size() - _rowsets_idx); + auto fill_idx_begin = _rowsets_idx; + auto fill_idx_end = _rowsets_idx + fill_rowsets_num; + std::vector datas(fill_rowsets_num); + // BACKEND_ID + { + int64_t src = backend_id_; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + datas[i - fill_idx_begin] = &src; + } + fill_dest_column_for_range(block, 0, datas); + } + // ROWSET_ID + { + std::string rowset_ids[fill_rowsets_num]; + StringRef strs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + rowset_ids[i - fill_idx_begin] = rowset->rowset_id().to_string(); + strs[i - fill_idx_begin] = StringRef(rowset_ids[i - fill_idx_begin].c_str(), + rowset_ids[i - fill_idx_begin].size()); + datas[i - fill_idx_begin] = strs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 1, datas); + } + // TABLET_ID + { + int64_t srcs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + srcs[i - fill_idx_begin] = rowset->rowset_meta()->tablet_id(); + datas[i - fill_idx_begin] = srcs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 2, datas); + } + // ROWSET_NUM_ROWS + { + int64_t srcs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + srcs[i - fill_idx_begin] = rowset->num_rows(); + datas[i - fill_idx_begin] = srcs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 3, datas); + } + // TXN_ID + { + int64_t srcs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + srcs[i - fill_idx_begin] = rowset->txn_id(); + datas[i - fill_idx_begin] = srcs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 4, datas); + } + // NUM_SEGMENTS + { + int64_t srcs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + srcs[i - fill_idx_begin] = rowset->num_segments(); + datas[i - fill_idx_begin] = srcs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 5, datas); + } + // START_VERSION + { + int64_t srcs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + srcs[i - fill_idx_begin] = rowset->start_version(); + datas[i - fill_idx_begin] = srcs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 6, datas); + } + // END_VERSION + { + int64_t srcs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + srcs[i - fill_idx_begin] = rowset->end_version(); + datas[i - fill_idx_begin] = srcs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 7, datas); + } + // INDEX_DISK_SIZE + { + size_t srcs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + srcs[i - fill_idx_begin] = rowset->index_disk_size(); + datas[i - fill_idx_begin] = srcs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 8, datas); + } + // DATA_DISK_SIZE + { + size_t srcs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + srcs[i - fill_idx_begin] = rowset->data_disk_size(); + datas[i - fill_idx_begin] = srcs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 9, datas); + } + // CREATION_TIME + { + size_t srcs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + srcs[i - fill_idx_begin] = rowset->creation_time(); + datas[i - fill_idx_begin] = srcs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 10, datas); + } + // NEWEST_WRITE_TIMESTAMP + { + size_t srcs[fill_rowsets_num]; + for (int i = fill_idx_begin; i < fill_idx_end; ++i) { + RowsetSharedPtr rowset = rowsets_[i]; + srcs[i - fill_idx_begin] = rowset->newest_write_timestamp(); + datas[i - fill_idx_begin] = srcs + i - fill_idx_begin; + } + fill_dest_column_for_range(block, 11, datas); + } + _rowsets_idx += fill_rowsets_num; return Status::OK(); } } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.h b/be/src/exec/schema_scanner/schema_rowsets_scanner.h index dd642225fad51b..d4f8256d570004 100644 --- a/be/src/exec/schema_scanner/schema_rowsets_scanner.h +++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.h @@ -34,17 +34,19 @@ class SchemaRowsetsScanner : public SchemaScanner { Status start(RuntimeState* state) override; Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_all_rowsets(); + Status _get_all_rowsets(); // Status get_new_segments(); Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); private: - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; int64_t backend_id_ = 0; std::vector rowsets_; // used for traversing rowsets_ - int rowsets_idx_ = 0; + int _rowsets_idx = 0; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp index 196f796d36202e..db658502d44d03 100644 --- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaSchemaPrivilegesScanner::_s_tbls_columns[] = { +std::vector SchemaSchemaPrivilegesScanner::_s_tbls_columns = { // name, type, size, is_null {"GRANTEE", TYPE_VARCHAR, sizeof(StringValue), true}, {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringValue), true}, @@ -33,9 +33,7 @@ SchemaScanner::ColumnDesc SchemaSchemaPrivilegesScanner::_s_tbls_columns[] = { }; SchemaSchemaPrivilegesScanner::SchemaSchemaPrivilegesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _priv_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_SCHEMA_PRIVILEGES), _priv_index(0) {} SchemaSchemaPrivilegesScanner::~SchemaSchemaPrivilegesScanner() {} @@ -43,7 +41,7 @@ Status SchemaSchemaPrivilegesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); return Status::OK(); } @@ -112,7 +110,7 @@ Status SchemaSchemaPrivilegesScanner::fill_one_col(const std::string* src, MemPo return Status::OK(); } -Status SchemaSchemaPrivilegesScanner::get_new_table() { +Status SchemaSchemaPrivilegesScanner::_get_new_table() { TGetTablesParams table_params; if (nullptr != _param->wild) { table_params.__set_pattern(*(_param->wild)); @@ -153,4 +151,78 @@ Status SchemaSchemaPrivilegesScanner::get_next_row(Tuple* tuple, MemPool* pool, return fill_one_row(tuple, pool); } +Status SchemaSchemaPrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + *eos = true; + if (!_priv_result.privileges.size()) { + return Status::OK(); + } + return _fill_block_impl(block); +} + +Status SchemaSchemaPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto privileges_num = _priv_result.privileges.size(); + std::vector datas(privileges_num); + + // grantee + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 0, datas); + } + // catalog + // This value is always def. + { + std::string definer = "def"; + StringRef str = StringRef(definer.c_str(), definer.size()); + for (int i = 0; i < privileges_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 1, datas); + } + // schema + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.schema.c_str(), priv_status.schema.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 2, datas); + } + // privilege type + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.privilege_type.c_str(), + priv_status.privilege_type.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 3, datas); + } + // is grantable + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 4, datas); + } + return Status::OK(); +} + } // namespace doris \ No newline at end of file diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h index d2f7c0f9b6cd42..691a64a33a4207 100644 --- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h @@ -25,19 +25,21 @@ namespace doris { class SchemaSchemaPrivilegesScanner : public SchemaScanner { public: SchemaSchemaPrivilegesScanner(); - virtual ~SchemaSchemaPrivilegesScanner(); + ~SchemaSchemaPrivilegesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); + Status _get_new_table(); Status fill_one_row(Tuple* tuple, MemPool* pool); Status fill_one_col(const std::string* src, MemPool* pool, void* slot); + Status _fill_block_impl(vectorized::Block* block); int _priv_index; TListPrivilegesResult _priv_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp index 278e93ddb15326..37f3cc996c2393 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaSchemataScanner::_s_columns[] = { +std::vector SchemaSchemataScanner::_s_columns = { // name, type, size {"CATALOG_NAME", TYPE_VARCHAR, sizeof(StringValue), true}, {"SCHEMA_NAME", TYPE_VARCHAR, sizeof(StringValue), false}, @@ -33,12 +33,12 @@ SchemaScanner::ColumnDesc SchemaSchemataScanner::_s_columns[] = { }; SchemaSchemataScanner::SchemaSchemataScanner() - : SchemaScanner(_s_columns, sizeof(_s_columns) / sizeof(SchemaScanner::ColumnDesc)), - _db_index(0) {} + : SchemaScanner(_s_columns, TSchemaTableType::SCH_SCHEMATA), _db_index(0) {} SchemaSchemataScanner::~SchemaSchemataScanner() {} Status SchemaSchemataScanner::start(RuntimeState* state) { + SCOPED_TIMER(_get_db_timer); if (!_is_init) { return Status::InternalError("used before initial."); } @@ -139,4 +139,72 @@ Status SchemaSchemataScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eo return fill_one_row(tuple, pool); } +Status SchemaSchemataScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before Initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + *eos = true; + if (!_db_result.dbs.size()) { + return Status::OK(); + } + return _fill_block_impl(block); +} + +Status SchemaSchemataScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto dbs_num = _db_result.dbs.size(); + std::vector null_datas(dbs_num, nullptr); + std::vector datas(dbs_num); + + // catalog + { + if (!_db_result.__isset.catalogs) { + fill_dest_column_for_range(block, 0, null_datas); + } else { + StringRef strs[dbs_num]; + for (int i = 0; i < dbs_num; ++i) { + strs[i] = StringRef(_db_result.catalogs[i].c_str(), _db_result.catalogs[i].size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 0, datas); + } + } + // schema + { + std::string db_names[dbs_num]; + StringRef strs[dbs_num]; + for (int i = 0; i < dbs_num; ++i) { + db_names[i] = SchemaHelper::extract_db_name(_db_result.dbs[i]); + strs[i] = StringRef(db_names[i].c_str(), db_names[i].size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 1, datas); + } + // DEFAULT_CHARACTER_SET_NAME + { + std::string src = "utf8"; + StringRef str = StringRef(src.c_str(), src.size()); + for (int i = 0; i < dbs_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 2, datas); + } + // DEFAULT_COLLATION_NAME + { + std::string src = "utf8_general_ci"; + StringRef str = StringRef(src.c_str(), src.size()); + for (int i = 0; i < dbs_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 3, datas); + } + // SQL_PATH + { fill_dest_column_for_range(block, 4, null_datas); } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.h b/be/src/exec/schema_scanner/schema_schemata_scanner.h index f6bc752021aaf0..a9aa1a00b7bb4a 100644 --- a/be/src/exec/schema_scanner/schema_schemata_scanner.h +++ b/be/src/exec/schema_scanner/schema_schemata_scanner.h @@ -25,17 +25,19 @@ namespace doris { class SchemaSchemataScanner : public SchemaScanner { public: SchemaSchemataScanner(); - virtual ~SchemaSchemataScanner(); + ~SchemaSchemataScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); int _db_index; TGetDbsResult _db_result; - static SchemaScanner::ColumnDesc _s_columns[]; + static std::vector _s_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_statistics_scanner.cpp b/be/src/exec/schema_scanner/schema_statistics_scanner.cpp index 51835ed22a1351..de5c8aca4d55be 100644 --- a/be/src/exec/schema_scanner/schema_statistics_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_statistics_scanner.cpp @@ -22,7 +22,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaStatisticsScanner::_s_cols_statistics[] = { +std::vector SchemaStatisticsScanner::_s_cols_statistics = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringValue), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringValue), false}, @@ -41,9 +41,7 @@ SchemaScanner::ColumnDesc SchemaStatisticsScanner::_s_cols_statistics[] = { {"COMMENT", TYPE_VARCHAR, sizeof(StringValue), true}, }; -SchemaStatisticsScanner::SchemaStatisticsScanner() - : SchemaScanner(_s_cols_statistics, - sizeof(_s_cols_statistics) / sizeof(SchemaScanner::ColumnDesc)) {} +SchemaStatisticsScanner::SchemaStatisticsScanner() : SchemaScanner(_s_cols_statistics) {} SchemaStatisticsScanner::~SchemaStatisticsScanner() {} diff --git a/be/src/exec/schema_scanner/schema_statistics_scanner.h b/be/src/exec/schema_scanner/schema_statistics_scanner.h index 7953a57987132c..e31c27277b2de2 100644 --- a/be/src/exec/schema_scanner/schema_statistics_scanner.h +++ b/be/src/exec/schema_scanner/schema_statistics_scanner.h @@ -23,9 +23,9 @@ namespace doris { class SchemaStatisticsScanner : public SchemaScanner { public: SchemaStatisticsScanner(); - virtual ~SchemaStatisticsScanner(); + ~SchemaStatisticsScanner() override; private: - static SchemaScanner::ColumnDesc _s_cols_statistics[]; + static std::vector _s_cols_statistics; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp index 372e721c287426..e033df865d7e54 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaTablePrivilegesScanner::_s_tbls_columns[] = { +std::vector SchemaTablePrivilegesScanner::_s_tbls_columns = { // name, type, size, is_null {"GRANTEE", TYPE_VARCHAR, sizeof(StringValue), true}, {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringValue), true}, @@ -34,9 +34,7 @@ SchemaScanner::ColumnDesc SchemaTablePrivilegesScanner::_s_tbls_columns[] = { }; SchemaTablePrivilegesScanner::SchemaTablePrivilegesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _priv_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_TABLE_PRIVILEGES), _priv_index(0) {} SchemaTablePrivilegesScanner::~SchemaTablePrivilegesScanner() {} @@ -44,7 +42,7 @@ Status SchemaTablePrivilegesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); return Status::OK(); } @@ -121,7 +119,8 @@ Status SchemaTablePrivilegesScanner::fill_one_col(const std::string* src, MemPoo return Status::OK(); } -Status SchemaTablePrivilegesScanner::get_new_table() { +Status SchemaTablePrivilegesScanner::_get_new_table() { + SCOPED_TIMER(_get_table_timer); TGetTablesParams table_params; if (nullptr != _param->wild) { table_params.__set_pattern(*(_param->wild)); @@ -162,4 +161,88 @@ Status SchemaTablePrivilegesScanner::get_next_row(Tuple* tuple, MemPool* pool, b return fill_one_row(tuple, pool); } +Status SchemaTablePrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + *eos = true; + if (!_priv_result.privileges.size()) { + return Status::OK(); + } + return _fill_block_impl(block); +} + +Status SchemaTablePrivilegesScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto privileges_num = _priv_result.privileges.size(); + std::vector datas(privileges_num); + + // grantee + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 0, datas); + } + // catalog + // This value is always def. + { + std::string definer = "def"; + StringRef str = StringRef(definer.c_str(), definer.size()); + for (int i = 0; i < privileges_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 1, datas); + } + // schema + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.schema.c_str(), priv_status.schema.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 2, datas); + } + // table name + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.table_name.c_str(), priv_status.table_name.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 3, datas); + } + // privilege type + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.privilege_type.c_str(), + priv_status.privilege_type.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 4, datas); + } + // is grantable + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 5, datas); + } + return Status::OK(); +} + } // namespace doris \ No newline at end of file diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h index d14e8005220832..7a382c1f3c7ad8 100644 --- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h @@ -25,19 +25,21 @@ namespace doris { class SchemaTablePrivilegesScanner : public SchemaScanner { public: SchemaTablePrivilegesScanner(); - virtual ~SchemaTablePrivilegesScanner(); + ~SchemaTablePrivilegesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); + Status _get_new_table(); Status fill_one_row(Tuple* tuple, MemPool* pool); Status fill_one_col(const std::string* src, MemPool* pool, void* slot); + Status _fill_block_impl(vectorized::Block* block); int _priv_index; TListPrivilegesResult _priv_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp index 8bf05b894f1d98..8597a450dc2f3b 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp @@ -17,13 +17,15 @@ #include "exec/schema_scanner/schema_tables_scanner.h" +#include "common/status.h" #include "exec/schema_scanner/schema_helper.h" #include "runtime/primitive_type.h" #include "runtime/string_value.h" +#include "vec/columns/column_complex.h" namespace doris { -SchemaScanner::ColumnDesc SchemaTablesScanner::_s_tbls_columns[] = { +std::vector SchemaTablesScanner::_s_tbls_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringValue), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringValue), false}, @@ -49,8 +51,7 @@ SchemaScanner::ColumnDesc SchemaTablesScanner::_s_tbls_columns[] = { }; SchemaTablesScanner::SchemaTablesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_TABLES), _db_index(0), _table_index(0) {} @@ -60,6 +61,7 @@ Status SchemaTablesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } + SCOPED_TIMER(_get_db_timer); TGetDbsParams db_params; if (nullptr != _param->db) { db_params.__set_pattern(*(_param->db)); @@ -257,7 +259,8 @@ Status SchemaTablesScanner::fill_one_row(Tuple* tuple, MemPool* pool) { return Status::OK(); } -Status SchemaTablesScanner::get_new_table() { +Status SchemaTablesScanner::_get_new_table() { + SCOPED_TIMER(_get_table_timer); TGetTablesParams table_params; table_params.__set_db(_db_result.dbs[_db_index]); if (_db_result.__isset.catalogs) { @@ -297,7 +300,7 @@ Status SchemaTablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) } while (_table_index >= _table_result.tables.size()) { if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); } else { *eos = true; return Status::OK(); @@ -307,4 +310,227 @@ Status SchemaTablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) return fill_one_row(tuple, pool); } +Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto table_num = _table_result.tables.size(); + std::vector null_datas(table_num, nullptr); + std::vector datas(table_num); + + // catalog + { + if (_db_result.__isset.catalogs) { + std::string catalog_name = _db_result.catalogs[_db_index - 1]; + StringRef str_slot = StringRef(catalog_name.c_str(), catalog_name.size()); + for (int i = 0; i < table_num; ++i) { + datas[i] = &str_slot; + } + fill_dest_column_for_range(block, 0, datas); + } else { + fill_dest_column_for_range(block, 0, null_datas); + } + } + // schema + { + std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); + StringRef str_slot = StringRef(db_name.c_str(), db_name.size()); + for (int i = 0; i < table_num; ++i) { + datas[i] = &str_slot; + } + fill_dest_column_for_range(block, 1, datas); + } + // name + { + StringRef strs[table_num]; + for (int i = 0; i < table_num; ++i) { + const std::string* src = &_table_result.tables[i].name; + strs[i] = StringRef(src->c_str(), src->size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 2, datas); + } + // type + { + StringRef strs[table_num]; + for (int i = 0; i < table_num; ++i) { + const std::string* src = &_table_result.tables[i].type; + strs[i] = StringRef(src->c_str(), src->size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 3, datas); + } + // engine + { + StringRef strs[table_num]; + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.engine) { + const std::string* src = &tbl_status.engine; + strs[i] = StringRef(src->c_str(), src->size()); + datas[i] = strs + i; + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 4, datas); + } + // version + { fill_dest_column_for_range(block, 5, null_datas); } + // row_format + { fill_dest_column_for_range(block, 6, null_datas); } + // rows + { + int64_t srcs[table_num]; + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.rows) { + srcs[i] = tbl_status.rows; + datas[i] = srcs + i; + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 7, datas); + } + // avg_row_length + { + int64_t srcs[table_num]; + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.avg_row_length) { + srcs[i] = tbl_status.avg_row_length; + datas[i] = srcs + i; + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 8, datas); + } + // data_length + { + int64_t srcs[table_num]; + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.avg_row_length) { + srcs[i] = tbl_status.data_length; + datas[i] = srcs + i; + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 9, datas); + } + // max_data_length + { fill_dest_column_for_range(block, 10, null_datas); } + // index_length + { fill_dest_column_for_range(block, 11, null_datas); } + // data_free + { fill_dest_column_for_range(block, 12, null_datas); } + // auto_increment + { fill_dest_column_for_range(block, 13, null_datas); } + // creation_time + { + DateTimeValue srcs[table_num]; + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.create_time) { + int64_t create_time = tbl_status.create_time; + if (create_time <= 0) { + datas[i] = nullptr; + } else { + srcs[i].from_unixtime(create_time, TimezoneUtils::default_time_zone); + datas[i] = srcs + i; + } + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 14, datas); + } + // update_time + { + DateTimeValue srcs[table_num]; + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.update_time) { + int64_t update_time = tbl_status.update_time; + if (update_time <= 0) { + datas[i] = nullptr; + } else { + srcs[i].from_unixtime(update_time, TimezoneUtils::default_time_zone); + datas[i] = srcs + i; + } + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 15, datas); + } + // check_time + { + DateTimeValue srcs[table_num]; + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.last_check_time) { + int64_t check_time = tbl_status.last_check_time; + if (check_time <= 0) { + datas[i] = nullptr; + } else { + srcs[i].from_unixtime(check_time, TimezoneUtils::default_time_zone); + datas[i] = srcs + i; + } + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 16, datas); + } + // collation + { + StringRef strs[table_num]; + for (int i = 0; i < table_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + if (tbl_status.__isset.collation) { + const std::string* src = &tbl_status.collation; + strs[i] = StringRef(src->c_str(), src->size()); + datas[i] = strs + i; + } else { + datas[i] = nullptr; + } + } + fill_dest_column_for_range(block, 17, datas); + } + // checksum + { fill_dest_column_for_range(block, 18, null_datas); } + // create_options + { fill_dest_column_for_range(block, 19, null_datas); } + // create_comment + { + StringRef strs[table_num]; + for (int i = 0; i < table_num; ++i) { + const std::string* src = &_table_result.tables[i].comment; + strs[i] = StringRef(src->c_str(), src->size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 20, datas); + } + return Status::OK(); +} + +Status SchemaTablesScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(_get_new_table()); + } else { + *eos = true; + return Status::OK(); + } + *eos = false; + return _fill_block_impl(block); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h index ff957999339eea..f111f42c5104db 100644 --- a/be/src/exec/schema_scanner/schema_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_tables_scanner.h @@ -17,28 +17,32 @@ #pragma once +#include "common/status.h" #include "exec/schema_scanner.h" #include "gen_cpp/FrontendService_types.h" +#include "vec/core/block.h" namespace doris { class SchemaTablesScanner : public SchemaScanner { public: SchemaTablesScanner(); - virtual ~SchemaTablesScanner(); + ~SchemaTablesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); + Status _get_new_table(); Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); int _db_index; int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp index 533311319e9006..34fa6b90010b8a 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaUserPrivilegesScanner::_s_tbls_columns[] = { +std::vector SchemaUserPrivilegesScanner::_s_tbls_columns = { // name, type, size, is_null {"GRANTEE", TYPE_VARCHAR, sizeof(StringValue), true}, {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringValue), true}, @@ -32,9 +32,7 @@ SchemaScanner::ColumnDesc SchemaUserPrivilegesScanner::_s_tbls_columns[] = { }; SchemaUserPrivilegesScanner::SchemaUserPrivilegesScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), - _priv_index(0) {} + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_USER_PRIVILEGES), _priv_index(0) {} SchemaUserPrivilegesScanner::~SchemaUserPrivilegesScanner() {} @@ -42,7 +40,7 @@ Status SchemaUserPrivilegesScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); return Status::OK(); } @@ -103,7 +101,8 @@ Status SchemaUserPrivilegesScanner::fill_one_col(const std::string* src, MemPool return Status::OK(); } -Status SchemaUserPrivilegesScanner::get_new_table() { +Status SchemaUserPrivilegesScanner::_get_new_table() { + SCOPED_TIMER(_get_table_timer); TGetTablesParams table_params; if (nullptr != _param->wild) { table_params.__set_pattern(*(_param->wild)); @@ -144,4 +143,68 @@ Status SchemaUserPrivilegesScanner::get_next_row(Tuple* tuple, MemPool* pool, bo return fill_one_row(tuple, pool); } +Status SchemaUserPrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + *eos = true; + if (!_priv_result.privileges.size()) { + return Status::OK(); + } + return _fill_block_impl(block); +} + +Status SchemaUserPrivilegesScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto privileges_num = _priv_result.privileges.size(); + std::vector datas(privileges_num); + + // grantee + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.grantee.c_str(), priv_status.grantee.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 0, datas); + } + // catalog + // This value is always def. + { + std::string definer = "def"; + StringRef str = StringRef(definer.c_str(), definer.size()); + for (int i = 0; i < privileges_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 1, datas); + } + // privilege type + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.privilege_type.c_str(), + priv_status.privilege_type.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 2, datas); + } + // is grantable + { + StringRef strs[privileges_num]; + for (int i = 0; i < privileges_num; ++i) { + const TPrivilegeStatus& priv_status = _priv_result.privileges[i]; + strs[i] = StringRef(priv_status.is_grantable.c_str(), priv_status.is_grantable.size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 3, datas); + } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h index 2e2cbc80dafb61..54cc51bf701485 100644 --- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h +++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h @@ -25,19 +25,21 @@ namespace doris { class SchemaUserPrivilegesScanner : public SchemaScanner { public: SchemaUserPrivilegesScanner(); - virtual ~SchemaUserPrivilegesScanner(); + ~SchemaUserPrivilegesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); + Status _get_new_table(); Status fill_one_row(Tuple* tuple, MemPool* pool); Status fill_one_col(const std::string* src, MemPool* pool, void* slot); + Status _fill_block_impl(vectorized::Block* block); int _priv_index; TListPrivilegesResult _priv_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.cpp b/be/src/exec/schema_scanner/schema_variables_scanner.cpp index 69842d81f00bd3..c4d886668dcbbb 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_variables_scanner.cpp @@ -24,16 +24,14 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaVariablesScanner::_s_vars_columns[] = { +std::vector SchemaVariablesScanner::_s_vars_columns = { // name, type, size {"VARIABLE_NAME", TYPE_VARCHAR, sizeof(StringValue), false}, {"VARIABLE_VALUE", TYPE_VARCHAR, sizeof(StringValue), false}, }; SchemaVariablesScanner::SchemaVariablesScanner(TVarType::type type) - : SchemaScanner(_s_vars_columns, - sizeof(_s_vars_columns) / sizeof(SchemaScanner::ColumnDesc)), - _type(type) {} + : SchemaScanner(_s_vars_columns, TSchemaTableType::SCH_VARIABLES), _type(type) {} SchemaVariablesScanner::~SchemaVariablesScanner() {} @@ -105,4 +103,48 @@ Status SchemaVariablesScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* e return fill_one_row(tuple, pool); } +Status SchemaVariablesScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("call this before initial."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("invalid parameter."); + } + + *eos = true; + if (_var_result.variables.empty()) { + return Status::OK(); + } + return _fill_block_impl(block); +} + +Status SchemaVariablesScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto row_num = _var_result.variables.size(); + std::vector datas(row_num); + // variables names + { + StringRef strs[row_num]; + int idx = 0; + for (auto& it : _var_result.variables) { + strs[idx] = StringRef(it.first.c_str(), it.first.size()); + datas[idx] = strs + idx; + ++idx; + } + fill_dest_column_for_range(block, 0, datas); + } + // value + { + StringRef strs[row_num]; + int idx = 0; + for (auto& it : _var_result.variables) { + strs[idx] = StringRef(it.second.c_str(), it.second.size()); + datas[idx] = strs + idx; + ++idx; + } + fill_dest_column_for_range(block, 1, datas); + } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.h b/be/src/exec/schema_scanner/schema_variables_scanner.h index 25daaf0e4f9c43..11cd4b5252b432 100644 --- a/be/src/exec/schema_scanner/schema_variables_scanner.h +++ b/be/src/exec/schema_scanner/schema_variables_scanner.h @@ -28,10 +28,11 @@ namespace doris { class SchemaVariablesScanner : public SchemaScanner { public: SchemaVariablesScanner(TVarType::type type); - virtual ~SchemaVariablesScanner(); + ~SchemaVariablesScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: struct VariableStruct { @@ -40,8 +41,9 @@ class SchemaVariablesScanner : public SchemaScanner { }; Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); - static SchemaScanner::ColumnDesc _s_vars_columns[]; + static std::vector _s_vars_columns; TShowVariableResult _var_result; TVarType::type _type; diff --git a/be/src/exec/schema_scanner/schema_views_scanner.cpp b/be/src/exec/schema_scanner/schema_views_scanner.cpp index 4df4d1de3175a7..c8f93f8c560850 100644 --- a/be/src/exec/schema_scanner/schema_views_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_views_scanner.cpp @@ -23,7 +23,7 @@ namespace doris { -SchemaScanner::ColumnDesc SchemaViewsScanner::_s_tbls_columns[] = { +std::vector SchemaViewsScanner::_s_tbls_columns = { // name, type, size, is_null {"TABLE_CATALOG", TYPE_VARCHAR, sizeof(StringValue), true}, {"TABLE_SCHEMA", TYPE_VARCHAR, sizeof(StringValue), false}, @@ -38,8 +38,7 @@ SchemaScanner::ColumnDesc SchemaViewsScanner::_s_tbls_columns[] = { }; SchemaViewsScanner::SchemaViewsScanner() - : SchemaScanner(_s_tbls_columns, - sizeof(_s_tbls_columns) / sizeof(SchemaScanner::ColumnDesc)), + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_VIEWS), _db_index(0), _table_index(0) {} @@ -49,6 +48,7 @@ Status SchemaViewsScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } + SCOPED_TIMER(_get_db_timer); TGetDbsParams db_params; if (nullptr != _param->db) { db_params.__set_pattern(*(_param->db)); @@ -186,7 +186,8 @@ Status SchemaViewsScanner::fill_one_row(Tuple* tuple, MemPool* pool) { return Status::OK(); } -Status SchemaViewsScanner::get_new_table() { +Status SchemaViewsScanner::_get_new_table() { + SCOPED_TIMER(_get_table_timer); TGetTablesParams table_params; table_params.__set_db(_db_result.dbs[_db_index++]); if (nullptr != _param->wild) { @@ -223,7 +224,7 @@ Status SchemaViewsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) } while (_table_index >= _table_result.tables.size()) { if (_db_index < _db_result.dbs.size()) { - RETURN_IF_ERROR(get_new_table()); + RETURN_IF_ERROR(_get_new_table()); } else { *eos = true; return Status::OK(); @@ -233,4 +234,114 @@ Status SchemaViewsScanner::get_next_row(Tuple* tuple, MemPool* pool, bool* eos) return fill_one_row(tuple, pool); } +Status SchemaViewsScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(_get_new_table()); + } else { + *eos = true; + return Status::OK(); + } + *eos = false; + return _fill_block_impl(block); +} + +Status SchemaViewsScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto tables_num = _table_result.tables.size(); + std::vector null_datas(tables_num, nullptr); + std::vector datas(tables_num); + + // catalog + { fill_dest_column_for_range(block, 0, null_datas); } + // schema + { + std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); + StringRef str = StringRef(db_name.c_str(), db_name.size()); + for (int i = 0; i < tables_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 1, datas); + } + // name + { + StringRef strs[tables_num]; + for (int i = 0; i < tables_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + const std::string* src = &tbl_status.name; + strs[i] = StringRef(src->c_str(), src->size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 2, datas); + } + // definition + { + StringRef strs[tables_num]; + for (int i = 0; i < tables_num; ++i) { + const TTableStatus& tbl_status = _table_result.tables[i]; + const std::string* src = &tbl_status.ddl_sql; + strs[i] = StringRef(src->c_str(), src->length()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 3, datas); + } + // check_option + { + const std::string check_option = "NONE"; + StringRef str = StringRef(check_option.c_str(), check_option.length()); + for (int i = 0; i < tables_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 4, datas); + } + // is_updatable + { + // This is from views in mysql + const std::string is_updatable = "NO"; + StringRef str = StringRef(is_updatable.c_str(), is_updatable.length()); + for (int i = 0; i < tables_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 5, datas); + } + // definer + { + // This is from views in mysql + const std::string definer = "root@%"; + StringRef str = StringRef(definer.c_str(), definer.length()); + for (int i = 0; i < tables_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 6, datas); + } + // security_type + { + // This is from views in mysql + const std::string security_type = "DEFINER"; + StringRef str = StringRef(security_type.c_str(), security_type.length()); + for (int i = 0; i < tables_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 7, datas); + } + // character_set_client + { + // This is from views in mysql + const std::string encoding = "utf8"; + StringRef str = StringRef(encoding.c_str(), encoding.length()); + for (int i = 0; i < tables_num; ++i) { + datas[i] = &str; + } + fill_dest_column_for_range(block, 8, datas); + } + // collation_connection + { fill_dest_column_for_range(block, 9, null_datas); } + return Status::OK(); +} + } // namespace doris diff --git a/be/src/exec/schema_scanner/schema_views_scanner.h b/be/src/exec/schema_scanner/schema_views_scanner.h index 3222b67e897d59..284581a65fcfa0 100644 --- a/be/src/exec/schema_scanner/schema_views_scanner.h +++ b/be/src/exec/schema_scanner/schema_views_scanner.h @@ -25,20 +25,22 @@ namespace doris { class SchemaViewsScanner : public SchemaScanner { public: SchemaViewsScanner(); - virtual ~SchemaViewsScanner(); + ~SchemaViewsScanner() override; - virtual Status start(RuntimeState* state); - virtual Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos); + Status start(RuntimeState* state) override; + Status get_next_row(Tuple* tuple, MemPool* pool, bool* eos) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; private: - Status get_new_table(); + Status _get_new_table(); Status fill_one_row(Tuple* tuple, MemPool* pool); + Status _fill_block_impl(vectorized::Block* block); int _db_index; int _table_index; TGetDbsResult _db_result; TListTableStatusResult _table_result; - static SchemaScanner::ColumnDesc _s_tbls_columns[]; + static std::vector _s_tbls_columns; }; } // namespace doris diff --git a/be/src/vec/exec/vschema_scan_node.cpp b/be/src/vec/exec/vschema_scan_node.cpp index 31d3fc60ef81af..b4808f3af3c23d 100644 --- a/be/src/vec/exec/vschema_scan_node.cpp +++ b/be/src/vec/exec/vschema_scan_node.cpp @@ -17,16 +17,22 @@ #include "vec/exec/vschema_scan_node.h" +#include +#include + #include "exec/text_converter.h" #include "exec/text_converter.hpp" #include "gen_cpp/PlanNodes_types.h" +#include "runtime/descriptors.h" #include "runtime/row_batch.h" #include "runtime/runtime_state.h" #include "runtime/string_value.h" #include "runtime/tuple_row.h" #include "util/runtime_profile.h" #include "util/types.h" +#include "vec/columns/column.h" #include "vec/core/types.h" +#include "vec/data_types/data_type_factory.hpp" namespace doris::vectorized { VSchemaScanNode::VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, @@ -35,26 +41,13 @@ VSchemaScanNode::VSchemaScanNode(ObjectPool* pool, const TPlanNode& tnode, _is_init(false), _table_name(tnode.schema_scan_node.table_name), _tuple_id(tnode.schema_scan_node.tuple_id), - _src_tuple_desc(nullptr), _dest_tuple_desc(nullptr), _tuple_idx(0), _slot_num(0), _tuple_pool(nullptr), - _schema_scanner(nullptr), - _src_tuple(nullptr), - _src_single_tuple(nullptr), - _dest_single_tuple(nullptr) {} - -VSchemaScanNode::~VSchemaScanNode() { - delete[] reinterpret_cast(_src_tuple); - _src_tuple = nullptr; + _schema_scanner(nullptr) {} - delete[] reinterpret_cast(_src_single_tuple); - _src_single_tuple = nullptr; - - delete[] reinterpret_cast(_dest_single_tuple); - _dest_single_tuple = nullptr; -} +VSchemaScanNode::~VSchemaScanNode() = default; Status VSchemaScanNode::init(const TPlanNode& tnode, RuntimeState* state) { RETURN_IF_ERROR(ExecNode::init(tnode, state)); @@ -169,6 +162,10 @@ Status VSchemaScanNode::prepare(RuntimeState* state) { return Status::InternalError("Failed to get schema table descriptor."); } + // init schema scanner profile + _scanner_param.profile.reset(new RuntimeProfile("SchemaScanner")); + _runtime_profile->add_child(_scanner_param.profile.get(), true, nullptr); + // new one scanner _schema_scanner.reset(SchemaScanner::create(schema_table->schema_table_type())); @@ -177,71 +174,42 @@ Status VSchemaScanNode::prepare(RuntimeState* state) { } RETURN_IF_ERROR(_schema_scanner->init(&_scanner_param, _pool)); - // get column info from scanner - _src_tuple_desc = _schema_scanner->tuple_desc(); - - if (nullptr == _src_tuple_desc) { - return Status::InternalError("failed to get src schema tuple desc."); - } - - _src_tuple = - reinterpret_cast(new (std::nothrow) char[_src_tuple_desc->byte_size()]); - - if (nullptr == _src_tuple) { - return Status::InternalError("new src tuple failed."); - } - // if src tuple desc slots is zero, it's the dummy slots. - if (0 == _src_tuple_desc->slots().size()) { + const std::vector& columns_desc(_schema_scanner->get_column_desc()); + // if src columns size is zero, it's the dummy slots. + if (0 == columns_desc.size()) { _slot_num = 0; } // check if type is ok. - if (_slot_num > 0) { - _index_map.resize(_slot_num); - } for (int i = 0; i < _slot_num; ++i) { // TODO(zhaochun): Is this slow? int j = 0; - for (; j < _src_tuple_desc->slots().size(); ++j) { - if (boost::iequals(_dest_tuple_desc->slots()[i]->col_name(), - _src_tuple_desc->slots()[j]->col_name())) { + for (; j < columns_desc.size(); ++j) { + if (boost::iequals(_dest_tuple_desc->slots()[i]->col_name(), columns_desc[j].name)) { break; } } - if (j >= _src_tuple_desc->slots().size()) { + if (j >= columns_desc.size()) { LOG(WARNING) << "no match column for this column(" << _dest_tuple_desc->slots()[i]->col_name() << ")"; return Status::InternalError("no match column for this column."); } - if (_src_tuple_desc->slots()[j]->type().type != _dest_tuple_desc->slots()[i]->type().type) { - LOG(WARNING) << "schema not match. input is " << _src_tuple_desc->slots()[j]->col_name() - << "(" << _src_tuple_desc->slots()[j]->type() << ") and output is " + if (columns_desc[j].type != _dest_tuple_desc->slots()[i]->type().type) { + LOG(WARNING) << "schema not match. input is " << columns_desc[j].name << "(" + << columns_desc[j].type << ") and output is " << _dest_tuple_desc->slots()[i]->col_name() << "(" << _dest_tuple_desc->slots()[i]->type() << ")"; return Status::InternalError("schema not match."); } - _index_map[i] = j; } // TODO(marcel): add int _tuple_idx indexed by TupleId somewhere in runtime_state.h _tuple_idx = 0; _is_init = true; - _src_single_tuple = - reinterpret_cast(new (std::nothrow) char[_src_tuple_desc->byte_size()]); - if (nullptr == _src_single_tuple) { - return Status::InternalError("new src single tuple failed."); - } - - _dest_single_tuple = - reinterpret_cast(new (std::nothrow) char[_dest_tuple_desc->byte_size()]); - if (nullptr == _dest_single_tuple) { - return Status::InternalError("new desc single tuple failed."); - } - return Status::OK(); } @@ -257,74 +225,55 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, return Status::InternalError("used before initialize."); } RETURN_IF_CANCELLED(state); - std::vector columns(_slot_num); bool schema_eos = false; - + const std::vector& columns_desc(_schema_scanner->get_column_desc()); do { - bool mem_reuse = block->mem_reuse(); - DCHECK(block->rows() == 0); - - columns.resize(_slot_num); + block->clear(); for (int i = 0; i < _slot_num; ++i) { - if (mem_reuse) { - columns[i] = std::move(*block->get_by_position(i).column).mutate(); - } else { - columns[i] = _dest_tuple_desc->slots()[i]->get_empty_mutable_column(); - } + auto dest_slot_desc = _dest_tuple_desc->slots()[i]; + block->insert(ColumnWithTypeAndName(dest_slot_desc->get_empty_mutable_column(), + dest_slot_desc->get_data_type_ptr(), + dest_slot_desc->col_name())); + } + // src block columns desc is filled by schema_scanner->get_column_desc. + vectorized::Block src_block; + for (int i = 0; i < columns_desc.size(); ++i) { + TypeDescriptor descriptor(columns_desc[i].type); + auto data_type = + vectorized::DataTypeFactory::instance().create_data_type(descriptor, true); + src_block.insert(ColumnWithTypeAndName(data_type->create_column(), data_type, + columns_desc[i].name)); } while (true) { RETURN_IF_CANCELLED(state); // get all slots from schema table. - RETURN_IF_ERROR(_schema_scanner->get_next_row(_src_single_tuple, _tuple_pool.get(), - &schema_eos)); + RETURN_IF_ERROR(_schema_scanner->get_next_block(&src_block, &schema_eos)); + if (schema_eos) { *eos = true; break; } - // tuple project - project_tuple(); - for (int i = 0; i < _slot_num; ++i) { - auto slot_desc = _dest_tuple_desc->slots()[i]; - if (!slot_desc->is_materialized()) { - continue; - } - - if (_dest_single_tuple->is_null(slot_desc->null_indicator_offset())) { - if (slot_desc->is_nullable()) { - auto* nullable_column = - reinterpret_cast(columns[i].get()); - nullable_column->insert_data(nullptr, 0); - } else { - return Status::InternalError( - "nonnull column contains NULL. table={}, column={}", _table_name, - slot_desc->col_name()); - } - } else { - RETURN_IF_ERROR(write_slot_to_vectorized_column( - _dest_single_tuple->get_slot(slot_desc->tuple_offset()), slot_desc, - &columns[i])); - } - } - if (columns[0]->size() == state->batch_size()) { + if (src_block.rows() >= state->batch_size()) { break; } } - if (!columns.empty() && !columns[0]->empty()) { - auto n_columns = 0; - if (!mem_reuse) { - for (const auto slot_desc : _dest_tuple_desc->slots()) { - block->insert(ColumnWithTypeAndName(std::move(columns[n_columns++]), - slot_desc->get_data_type_ptr(), - slot_desc->col_name())); - } - } else { - columns.clear(); + + if (src_block.rows()) { + // block->check_number_of_rows(); + for (int i = 0; i < _slot_num; ++i) { + auto dest_slot_desc = _dest_tuple_desc->slots()[i]; + vectorized::MutableColumnPtr column_ptr = + std::move(*block->get_by_position(i).column).mutate(); + column_ptr->insert_range_from( + *src_block.get_by_name(dest_slot_desc->col_name()).column, 0, + src_block.rows()); } RETURN_IF_ERROR(VExprContext::filter_block(_vconjunct_ctx_ptr, block, _dest_tuple_desc->slots().size())); - VLOG_ROW << "VSchemaScanNode output rows: " << block->rows(); + VLOG_ROW << "VSchemaScanNode output rows: " << src_block.rows(); + src_block.clear(); } } while (block->rows() == 0 && !(*eos)); @@ -332,171 +281,6 @@ Status VSchemaScanNode::get_next(RuntimeState* state, vectorized::Block* block, return Status::OK(); } -Status VSchemaScanNode::write_slot_to_vectorized_column(void* slot, SlotDescriptor* slot_desc, - vectorized::MutableColumnPtr* column_ptr) { - vectorized::IColumn* col_ptr = column_ptr->get(); - if (slot_desc->is_nullable()) { - auto* nullable_column = reinterpret_cast(column_ptr->get()); - nullable_column->get_null_map_data().push_back(0); - col_ptr = &nullable_column->get_nested_column(); - } - switch (slot_desc->type().type) { - case TYPE_HLL: { - HyperLogLog* hll_slot = reinterpret_cast(slot); - reinterpret_cast(col_ptr)->get_data().emplace_back(*hll_slot); - break; - } - case TYPE_VARCHAR: - case TYPE_CHAR: - case TYPE_STRING: { - StringValue* str_slot = reinterpret_cast(slot); - reinterpret_cast(col_ptr)->insert_data(str_slot->ptr, - str_slot->len); - break; - } - - case TYPE_BOOLEAN: { - uint8_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_TINYINT: { - int8_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_SMALLINT: { - int16_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_INT: { - int32_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_BIGINT: { - int64_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_LARGEINT: { - __int128 num; - memcpy(&num, slot, sizeof(__int128)); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_FLOAT: { - float num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value( - num); - break; - } - - case TYPE_DOUBLE: { - double num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value( - num); - break; - } - - case TYPE_DATE: { - VecDateTimeValue value; - DateTimeValue* ts_slot = reinterpret_cast(slot); - value.convert_dt_to_vec_dt(ts_slot); - reinterpret_cast*>(col_ptr)->insert_data( - reinterpret_cast(&value), 0); - break; - } - - case TYPE_DATEV2: { - uint32_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_DATETIME: { - VecDateTimeValue value; - DateTimeValue* ts_slot = reinterpret_cast(slot); - value.convert_dt_to_vec_dt(ts_slot); - reinterpret_cast*>(col_ptr)->insert_data( - reinterpret_cast(&value), 0); - break; - } - - case TYPE_DATETIMEV2: { - uint32_t num = *reinterpret_cast(slot); - reinterpret_cast*>(col_ptr)->insert_value(num); - break; - } - - case TYPE_DECIMALV2: { - const Int128 num = (reinterpret_cast(slot))->value; - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - case TYPE_DECIMAL128I: { - const Int128 num = (reinterpret_cast(slot))->value; - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - case TYPE_DECIMAL32: { - const int32_t num = *reinterpret_cast(slot); - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - case TYPE_DECIMAL64: { - const int64_t num = *reinterpret_cast(slot); - reinterpret_cast(col_ptr)->insert_data( - reinterpret_cast(&num), 0); - break; - } - - default: { - DCHECK(false) << "bad slot type: " << slot_desc->type(); - std::stringstream ss; - ss << "Fail to convert schema type:'" << slot_desc->type() << " on column:`" - << slot_desc->col_name() + "`"; - return Status::InternalError(ss.str()); - } - } - - return Status::OK(); -} - -void VSchemaScanNode::project_tuple() { - memset(_dest_single_tuple, 0, _dest_tuple_desc->num_null_bytes()); - - for (int i = 0; i < _slot_num; ++i) { - if (!_dest_tuple_desc->slots()[i]->is_materialized()) { - continue; - } - int j = _index_map[i]; - - if (_src_single_tuple->is_null(_src_tuple_desc->slots()[j]->null_indicator_offset())) { - _dest_single_tuple->set_null(_dest_tuple_desc->slots()[i]->null_indicator_offset()); - } else { - void* dest_slot = - _dest_single_tuple->get_slot(_dest_tuple_desc->slots()[i]->tuple_offset()); - void* src_slot = - _src_single_tuple->get_slot(_src_tuple_desc->slots()[j]->tuple_offset()); - int slot_size = _src_tuple_desc->slots()[j]->type().get_slot_size(); - memcpy(dest_slot, src_slot, slot_size); - } - } -} - Status VSchemaScanNode::close(RuntimeState* state) { if (is_closed()) { return Status::OK(); diff --git a/be/src/vec/exec/vschema_scan_node.h b/be/src/vec/exec/vschema_scan_node.h index 3fadb242f50f59..63a9793d765794 100644 --- a/be/src/vec/exec/vschema_scan_node.h +++ b/be/src/vec/exec/vschema_scan_node.h @@ -40,7 +40,7 @@ class VSchemaScanNode : public ScanNode { Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) override { return Status::NotSupported("Not Implemented VSchemaScanNode Node::get_next scalar"); } - virtual Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos) override; + Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos) override; // Prepare conjuncts, create Schema columns to slots mapping // initialize schema_scanner @@ -64,8 +64,6 @@ class VSchemaScanNode : public ScanNode { // Tuple id resolved in prepare() to set _tuple_desc; TupleId _tuple_id; - // Descriptor of tuples read from schema table. - const TupleDescriptor* _src_tuple_desc; // Descriptor of dest tuples const TupleDescriptor* _dest_tuple_desc; // Tuple index in tuple row. @@ -76,16 +74,6 @@ class VSchemaScanNode : public ScanNode { std::unique_ptr _tuple_pool; // Jni helper for scanning an schema table. std::unique_ptr _schema_scanner; - // Current tuple. - doris::Tuple* _src_tuple; - // Map from index in slots to column of schema table. - std::vector _index_map; - - Status write_slot_to_vectorized_column(void* slot, SlotDescriptor* slot_desc, - vectorized::MutableColumnPtr* col_ptr); - void project_tuple(); - doris::Tuple* _src_single_tuple; - doris::Tuple* _dest_single_tuple; }; } // namespace vectorized } // namespace doris \ No newline at end of file diff --git a/be/test/exec/schema_scanner_test.cpp b/be/test/exec/schema_scanner_test.cpp index 4bb9d6ff8d7177..4602c2282b729e 100644 --- a/be/test/exec/schema_scanner_test.cpp +++ b/be/test/exec/schema_scanner_test.cpp @@ -46,7 +46,7 @@ class SchemaScannerTest : public testing::Test { std::string _wild; }; -SchemaScanner::ColumnDesc s_test_columns[] = { +std::vector s_test_columns = { // name, type, size, is_null {"Name", TYPE_VARCHAR, sizeof(StringValue), false}, {"Location", TYPE_VARCHAR, sizeof(StringValue), false}, @@ -56,8 +56,7 @@ SchemaScanner::ColumnDesc s_test_columns[] = { char g_tuple_buf[10000]; // enough for tuple TEST_F(SchemaScannerTest, normal_use) { - SchemaScanner scanner(s_test_columns, - sizeof(s_test_columns) / sizeof(SchemaScanner::ColumnDesc)); + SchemaScanner scanner(s_test_columns); Status status = scanner.init(&_param, &_obj_pool); EXPECT_TRUE(status.ok()); status = scanner.init(&_param, &_obj_pool); @@ -73,8 +72,7 @@ TEST_F(SchemaScannerTest, normal_use) { EXPECT_TRUE(status.ok()); } TEST_F(SchemaScannerTest, input_fail) { - SchemaScanner scanner(s_test_columns, - sizeof(s_test_columns) / sizeof(SchemaScanner::ColumnDesc)); + SchemaScanner scanner(s_test_columns); Status status = scanner.init(&_param, &_obj_pool); EXPECT_TRUE(status.ok()); status = scanner.init(&_param, &_obj_pool); @@ -89,13 +87,12 @@ TEST_F(SchemaScannerTest, input_fail) { EXPECT_FALSE(status.ok()); } TEST_F(SchemaScannerTest, invalid_param) { - SchemaScanner scanner(nullptr, sizeof(s_test_columns) / sizeof(SchemaScanner::ColumnDesc)); + SchemaScanner scanner(nullptr); Status status = scanner.init(&_param, &_obj_pool); EXPECT_FALSE(status.ok()); } TEST_F(SchemaScannerTest, no_init_use) { - SchemaScanner scanner(s_test_columns, - sizeof(s_test_columns) / sizeof(SchemaScanner::ColumnDesc)); + SchemaScanner scanner(s_test_columns); Status status = scanner.start((RuntimeState*)1); EXPECT_FALSE(status.ok()); const TupleDescriptor* tuple_desc = scanner.tuple_desc(); diff --git a/be/test/vec/core/block_test.cpp b/be/test/vec/core/block_test.cpp index c0bbd1faf4adea..d6b718905d433e 100644 --- a/be/test/vec/core/block_test.cpp +++ b/be/test/vec/core/block_test.cpp @@ -50,7 +50,7 @@ namespace doris { using vectorized::Int32; TEST(BlockTest, RowBatchCovertToBlock) { - SchemaScanner::ColumnDesc column_descs[] = { + std::vector column_descs = { {"k1", TYPE_SMALLINT, sizeof(int16_t), true}, {"k2", TYPE_INT, sizeof(int32_t), false}, {"k3", TYPE_DOUBLE, sizeof(double), false}, @@ -60,8 +60,7 @@ TEST(BlockTest, RowBatchCovertToBlock) { {"k7", TYPE_DATETIME, sizeof(int64_t), false}, {"k8", TYPE_DATEV2, sizeof(uint32_t), false}}; - SchemaScanner schema_scanner(column_descs, - sizeof(column_descs) / sizeof(SchemaScanner::ColumnDesc)); + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool); diff --git a/be/test/vec/exec/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/parquet/parquet_thrift_test.cpp index d1f59e923404ed..377298bd4d9d39 100644 --- a/be/test/vec/exec/parquet/parquet_thrift_test.cpp +++ b/be/test/vec/exec/parquet/parquet_thrift_test.cpp @@ -224,7 +224,7 @@ static Status get_column_values(FileReader* file_reader, tparquet::ColumnChunk* static void create_block(std::unique_ptr& block) { // Current supported column type: - SchemaScanner::ColumnDesc column_descs[] = { + std::vector column_descs = { {"tinyint_col", TYPE_TINYINT, sizeof(int8_t), true}, {"smallint_col", TYPE_SMALLINT, sizeof(int16_t), true}, {"int_col", TYPE_INT, sizeof(int32_t), true}, @@ -243,8 +243,7 @@ static void create_block(std::unique_ptr& block) { {"date_col", TYPE_DATE, sizeof(DateTimeValue), true}, {"date_v2_col", TYPE_DATEV2, sizeof(uint32_t), true}, {"timestamp_v2_col", TYPE_DATETIMEV2, sizeof(DateTimeValue), true, 18, 0}}; - SchemaScanner schema_scanner(column_descs, - sizeof(column_descs) / sizeof(SchemaScanner::ColumnDesc)); + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool); @@ -342,7 +341,7 @@ TEST_F(ParquetThriftReaderTest, dict_decoder) { } TEST_F(ParquetThriftReaderTest, group_reader) { - SchemaScanner::ColumnDesc column_descs[] = { + std::vector column_descs = { {"tinyint_col", TYPE_TINYINT, sizeof(int8_t), true}, {"smallint_col", TYPE_SMALLINT, sizeof(int16_t), true}, {"int_col", TYPE_INT, sizeof(int32_t), true}, @@ -357,8 +356,7 @@ TEST_F(ParquetThriftReaderTest, group_reader) { {"char_col", TYPE_CHAR, sizeof(StringValue), true}, {"varchar_col", TYPE_VARCHAR, sizeof(StringValue), true}, {"date_col", TYPE_DATE, sizeof(DateTimeValue), true}}; - int num_cols = sizeof(column_descs) / sizeof(SchemaScanner::ColumnDesc); - SchemaScanner schema_scanner(column_descs, num_cols); + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool); diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp index 9ebf5612a864f9..254b1a37710031 100644 --- a/be/test/vec/exprs/vexpr_test.cpp +++ b/be/test/vec/exprs/vexpr_test.cpp @@ -86,8 +86,9 @@ TEST(TEST_VEXPR, ABSTEST) { TEST(TEST_VEXPR, ABSTEST2) { using namespace doris; - SchemaScanner::ColumnDesc column_descs[] = {{"k1", TYPE_INT, sizeof(int32_t), false}}; - SchemaScanner schema_scanner(column_descs, 1); + std::vector column_descs = { + {"k1", TYPE_INT, sizeof(int32_t), false}}; + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool); diff --git a/be/test/vec/function/function_comparison_test.cpp b/be/test/vec/function/function_comparison_test.cpp index 0c2e53961d62bf..87ac4ad682c8cb 100644 --- a/be/test/vec/function/function_comparison_test.cpp +++ b/be/test/vec/function/function_comparison_test.cpp @@ -29,10 +29,11 @@ namespace doris { TEST(ComparisonTest, ComparisonFunctionTest) { - SchemaScanner::ColumnDesc column_descs[] = {{"k1", TYPE_SMALLINT, sizeof(int16_t), false}, - {"k2", TYPE_INT, sizeof(int32_t), false}, - {"k3", TYPE_DOUBLE, sizeof(double), false}}; - SchemaScanner schema_scanner(column_descs, 3); + std::vector column_descs = { + {"k1", TYPE_SMALLINT, sizeof(int16_t), false}, + {"k2", TYPE_INT, sizeof(int32_t), false}, + {"k3", TYPE_DOUBLE, sizeof(double), false}}; + SchemaScanner schema_scanner(column_descs); ObjectPool object_pool; SchemaScannerParam param; schema_scanner.init(¶m, &object_pool); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index b00cfab65ac12d..d5d276876607e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -73,6 +73,8 @@ import org.apache.doris.thrift.TColumnDesc; import org.apache.doris.thrift.TDescribeTableParams; import org.apache.doris.thrift.TDescribeTableResult; +import org.apache.doris.thrift.TDescribeTablesParams; +import org.apache.doris.thrift.TDescribeTablesResult; import org.apache.doris.thrift.TExecPlanFragmentParams; import org.apache.doris.thrift.TFeResult; import org.apache.doris.thrift.TFetchResourceResult; @@ -488,6 +490,79 @@ public TDescribeTableResult describeTable(TDescribeTableParams params) throws TE return result; } + @Override + public TDescribeTablesResult describeTables(TDescribeTablesParams params) throws TException { + LOG.debug("get desc tables request: {}", params); + TDescribeTablesResult result = new TDescribeTablesResult(); + List columns = Lists.newArrayList(); + List tablesOffset = Lists.newArrayList(); + List tables = params.getTablesName(); + result.setColumns(columns); + result.setTablesOffset(tablesOffset); + + // database privs should be checked in analysis phrase + UserIdentity currentUser = null; + if (params.isSetCurrentUserIdent()) { + currentUser = UserIdentity.fromThrift(params.current_user_ident); + } else { + currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip); + } + for (String tableName : tables) { + if (!Env.getCurrentEnv().getAuth() + .checkTblPriv(currentUser, params.db, tableName, PrivPredicate.SHOW)) { + return result; + } + } + + String catalogName = Strings.isNullOrEmpty(params.catalog) ? InternalCatalog.INTERNAL_CATALOG_NAME + : params.catalog; + DatabaseIf db = Env.getCurrentEnv().getCatalogMgr() + .getCatalogOrException(catalogName, catalog -> new TException("Unknown catalog " + catalog)) + .getDbNullable(params.db); + if (db != null) { + for (String tableName : tables) { + TableIf table = db.getTableNullableIfException(tableName); + if (table != null) { + table.readLock(); + try { + List baseSchema = table.getBaseSchemaOrEmpty(); + for (Column column : baseSchema) { + final TColumnDesc desc = new TColumnDesc(column.getName(), column.getDataType().toThrift()); + final Integer precision = column.getOriginType().getPrecision(); + if (precision != null) { + desc.setColumnPrecision(precision); + } + final Integer columnLength = column.getOriginType().getColumnSize(); + if (columnLength != null) { + desc.setColumnLength(columnLength); + } + final Integer decimalDigits = column.getOriginType().getDecimalDigits(); + if (decimalDigits != null) { + desc.setColumnScale(decimalDigits); + } + desc.setIsAllowNull(column.isAllowNull()); + final TColumnDef colDef = new TColumnDef(desc); + final String comment = column.getComment(); + if (comment != null) { + colDef.setComment(comment); + } + if (column.isKey()) { + if (table instanceof OlapTable) { + desc.setColumnKey(((OlapTable) table).getKeysType().toMetadata()); + } + } + columns.add(colDef); + } + } finally { + table.readUnlock(); + } + tablesOffset.add(columns.size()); + } + } + } + return result; + } + @Override public TShowVariableResult showVariables(TShowVariableRequest params) throws TException { TShowVariableResult result = new TShowVariableResult(); diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 32c73a56634c6c..9a68230014c594 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -77,6 +77,25 @@ struct TDescribeTableResult { 1: required list columns } +// Arguments to DescribeTables, which returns a list of column descriptors for +// given tables +struct TDescribeTablesParams { + 1: optional string db + 2: required list tables_name + 3: optional string user // deprecated + 4: optional string user_ip // deprecated + 5: optional Types.TUserIdentity current_user_ident // to replace the user and user ip + 6: optional bool show_hidden_columns = false + 7: optional string catalog +} + +// Results of a call to describeTable() +struct TDescribeTablesResult { + // tables_offset means that the offset for each table in columns + 1: required list tables_offset + 2: required list columns +} + struct TShowVariableRequest { 1: required i64 threadId 2: required Types.TVarType varType @@ -719,6 +738,7 @@ service FrontendService { TGetDbsResult getDbNames(1: TGetDbsParams params) TGetTablesResult getTableNames(1: TGetTablesParams params) TDescribeTableResult describeTable(1: TDescribeTableParams params) + TDescribeTablesResult describeTables(1: TDescribeTablesParams params) TShowVariableResult showVariables(1: TShowVariableRequest params) TReportExecStatusResult reportExecStatus(1: TReportExecStatusParams params) diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out new file mode 100644 index 00000000000000..797e4a339857c3 --- /dev/null +++ b/regression-test/data/query_p0/system/test_query_sys_tables.out @@ -0,0 +1,78 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !backends -- +true + +-- !charsets -- +true + +-- !collations -- +true + +-- !columns -- +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 +internal aaa 1 varchar varchar(170) 170 +internal bbb 2 varchar varchar(20) 20 +internal ccc 3 int int(11) 10 +internal ddd 4 smallint smallint(6) 5 + +-- !schemata -- +internal test_query_sys_db_1 \N +internal test_query_sys_db_2 \N +internal test_query_sys_db_3 \N + +-- !tables -- +internal test_query_sys_tb_1 BASE TABLE 0 \N \N +internal test_query_sys_tb_2 BASE TABLE 0 \N \N +internal test_query_sys_tb_3 BASE TABLE 0 \N \N + +-- !session_variables -- +wait_timeout 30000 + +-- !global_variables -- +wait_timeout 31000 + +-- !user_privileges -- +'test_sys_tables'@'%' SELECT NO + +-- !views -- +test_view + +-- !sql -- +34000 + +-- !sql -- +34000 + +-- !sql -- +34000 + +-- !sql -- +34000 + +-- !sql -- +34000 + +-- !sql -- +34000 + +-- !sql -- +34000 + +-- !sql -- +34000 + +-- !sql -- +34000 + +-- !sql -- +34000 + +-- !sql -- +DUP diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy new file mode 100644 index 00000000000000..94d3950e8debb1 --- /dev/null +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_query_sys_tables", "query,p0") { + def dbName1 = "test_query_sys_db_1" + def dbName2 = "test_query_sys_db_2" + def dbName3 = "test_query_sys_db_3" + def tbName1 = "test_query_sys_tb_1" + def tbName2 = "test_query_sys_tb_2" + def tbName3 = "test_query_sys_tb_3" + sql("drop database IF EXISTS ${dbName1}") + sql("drop database IF EXISTS ${dbName2}") + sql("drop database IF EXISTS ${dbName3}") + + // test backends + sql("use information_schema") + qt_backends("select count(*) >= 1 from backends") + + // test charsets + sql("use information_schema") + qt_charsets("select count(*) >= 1 from character_sets") + + // test collations + sql("use information_schema") + qt_collations("select count(*) >= 1 from collations") + + // test columns + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName2}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName3}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use information_schema") + qt_columns("select TABLE_CATALOG, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE, COLUMN_TYPE, COLUMN_SIZE from columns where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'") + + // test files + // have no impl + + // test partitions + // have no impl + + // test rowsets + // have no tablet system table, add this later + + // test schemata + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + + sql("use information_schema") + qt_schemata("select CATALOG_NAME, SCHEMA_NAME, SQL_PATH from schemata where SCHEMA_NAME = '${dbName1}' or SCHEMA_NAME = '${dbName2}' or SCHEMA_NAME = '${dbName3}'"); + + // test statistics + // have no impl + + // test tables + // create test dbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("CREATE DATABASE IF NOT EXISTS ${dbName1}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName2}") + sql("CREATE DATABASE IF NOT EXISTS ${dbName3}") + // create test tbs + sql("use ${dbName1}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName2}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + sql("use ${dbName3}") + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + + sql("use information_schema") + qt_tables("select TABLE_CATALOG, TABLE_NAME, TABLE_TYPE, AVG_ROW_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH from tables where TABLE_SCHEMA = '${dbName1}' or TABLE_SCHEMA = '${dbName2}' or TABLE_SCHEMA = '${dbName3}'"); + + // test variables + // session_variables + sql("use information_schema") + sql("SET wait_timeout = 30000") + qt_session_variables("select VARIABLE_NAME, VARIABLE_VALUE from session_variables where VARIABLE_NAME = 'wait_timeout'") + + // global_variables + sql("use information_schema") + sql("SET GLOBAL wait_timeout = 31000") + qt_global_variables("select VARIABLE_NAME, VARIABLE_VALUE from global_variables where VARIABLE_NAME = 'wait_timeout'") + + // test user_privileges + sql("CREATE USER 'test_sys_tables'") + sql("GRANT SELECT_PRIV ON *.*.* TO 'test_sys_tables'") + sql("use information_schema") + qt_user_privileges """ + select GRANTEE, PRIVILEGE_TYPE, IS_GRANTABLE from user_privileges where GRANTEE regexp '^\\'test' + """ + sql("DROP USER 'test_sys_tables'") + + // test views + sql("use ${dbName1}") + sql """ + CREATE VIEW IF NOT EXISTS ${dbName1}.test_view (a) + AS + SELECT ccc as a FROM ${tbName1} + """ + sql("use information_schema") + qt_views("select TABLE_NAME, VIEW_DEFINITION from views where TABLE_SCHEMA = '${dbName1}'") + + // test a large amount of data + def dbPrefix = "db_query_sys_tables_with_lots_of_tables_" + def tablePrefix = "tb_query_sys_tables_with_lots_of_tables_" + + // create lots of dbs and tables to make rows in `information_schema.columns` + for (int i = 1; i <= 10; i++) { + def dbName = dbPrefix + i.toString() + sql "CREATE DATABASE IF NOT EXISTS `${dbName}`" + sql "USE `${dbName}`" + for (int j = 1; j <= 1000; j++) { + def tableName = tablePrefix + j.toString(); + sql """ + CREATE TABLE IF NOT EXISTS `${tableName}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(100) NOT NULL COMMENT "", + `ccc` varchar(170) NULL COMMENT "", + `ddd` varchar(120) NULL COMMENT "", + `eee` varchar(120) NULL COMMENT "", + `fff` varchar(130) NULL COMMENT "", + `ggg` varchar(170) NULL COMMENT "", + `hhh` varchar(170) NULL COMMENT "", + `jjj` varchar(170) NULL COMMENT "", + `kkk` varchar(170) NULL COMMENT "", + `lll` varchar(170) NULL COMMENT "", + `mmm` varchar(170) NULL COMMENT "", + `nnn` varchar(70) NULL COMMENT "", + `ooo` varchar(140) NULL COMMENT "", + `ppp` varchar(70) NULL COMMENT "", + `qqq` varchar(130) NULL COMMENT "", + `rrr` bigint(20) NULL COMMENT "", + `sss` bigint(20) NULL COMMENT "", + `ttt` decimal(24, 2) NULL COMMENT "", + `uuu` decimal(24, 2) NULL COMMENT "", + `vvv` decimal(24, 2) NULL COMMENT "", + `www` varchar(50) NULL COMMENT "", + `xxx` varchar(190) NULL COMMENT "", + `yyy` varchar(190) NULL COMMENT "", + `zzz` varchar(100) NULL COMMENT "", + `aa` bigint(20) NULL COMMENT "", + `bb` bigint(20) NULL COMMENT "", + `cc` bigint(20) NULL COMMENT "", + `dd` varchar(60) NULL COMMENT "", + `ee` varchar(60) NULL COMMENT "", + `ff` varchar(60) NULL COMMENT "", + `gg` varchar(50) NULL COMMENT "", + `hh` bigint(20) NULL COMMENT "", + `ii` bigint(20) NULL COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`aaa`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ) + """ + } + } + + for (int i = 1; i <= 10; i++) { + def dbName = dbPrefix + i.toString() + sql "USE information_schema" + qt_sql "SELECT COUNT(*) FROM `columns` WHERE TABLE_SCHEMA='${dbName}'" + } + + sql "USE information_schema" + qt_sql "SELECT COLUMN_KEY FROM `columns` WHERE TABLE_SCHEMA='db_query_sys_tables_with_lots_of_tables_1' and TABLE_NAME='tb_query_sys_tables_with_lots_of_tables_1' and COLUMN_NAME='aaa'" + + for (int i = 1; i <= 10; i++) { + def dbName = dbPrefix + i.toString() + sql "DROP DATABASE `${dbName}`" + } +} \ No newline at end of file