From 680a04a956df83956f4828c1351d7ebc3e8d35a2 Mon Sep 17 00:00:00 2001 From: changyuwei <2017501503@qq.com> Date: Tue, 8 Aug 2023 09:50:24 +0800 Subject: [PATCH 1/8] add information_schema.simple_tables for quickly get catlogs,db,table. --- be/src/exec/schema_scanner.cpp | 3 + be/src/exec/schema_scanner/schema_helper.cpp | 8 + be/src/exec/schema_scanner/schema_helper.h | 3 + .../schema_simple_tables_scanner.cpp | 227 ++++++++++++++++++ .../schema_simple_tables_scanner.h | 54 +++++ .../doris/analysis/SchemaTableType.java | 4 +- .../org/apache/doris/catalog/SchemaTable.java | 76 +++--- .../doris/datasource/ExternalCatalog.java | 1 + .../org/apache/doris/qe/SessionVariable.java | 10 + .../doris/service/FrontendServiceImpl.java | 102 +++++++- gensrc/thrift/Descriptors.thrift | 3 +- gensrc/thrift/FrontendService.thrift | 13 +- 12 files changed, 454 insertions(+), 50 deletions(-) create mode 100644 be/src/exec/schema_scanner/schema_simple_tables_scanner.cpp create mode 100644 be/src/exec/schema_scanner/schema_simple_tables_scanner.h diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index dce7cc5c0e66ed..9355bc1b194f3c 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -35,6 +35,7 @@ #include "exec/schema_scanner/schema_rowsets_scanner.h" #include "exec/schema_scanner/schema_schema_privileges_scanner.h" #include "exec/schema_scanner/schema_schemata_scanner.h" +#include "exec/schema_scanner/schema_simple_tables_scanner.h" #include "exec/schema_scanner/schema_table_privileges_scanner.h" #include "exec/schema_scanner/schema_tables_scanner.h" #include "exec/schema_scanner/schema_user_privileges_scanner.h" @@ -144,6 +145,8 @@ std::unique_ptr SchemaScanner::create(TSchemaTableType::type type return SchemaPartitionsScanner::create_unique(); case TSchemaTableType::SCH_ROWSETS: return SchemaRowsetsScanner::create_unique(); + case TSchemaTableType::SCH_SIMPLE_TABLES: + return SchemaSimpleTablesScanner::create_unique(); default: return SchemaDummyScanner::create_unique(); break; diff --git a/be/src/exec/schema_scanner/schema_helper.cpp b/be/src/exec/schema_scanner/schema_helper.cpp index 9dbc40886e1bc2..f811a2e628aa16 100644 --- a/be/src/exec/schema_scanner/schema_helper.cpp +++ b/be/src/exec/schema_scanner/schema_helper.cpp @@ -60,6 +60,14 @@ Status SchemaHelper::list_table_status(const std::string& ip, const int32_t port client->listTableStatus(*result, request); }); } +Status SchemaHelper::list_simple_table_status(const std::string& ip, const int32_t port, + const doris::TGetTablesParams& request, + TListSimpleTableStatusResult* result) { + return ThriftRpcHelper::rpc( + ip, port, [&request, &result](FrontendServiceConnection& client) { + client->listSimpleTableStatus(*result, request); + }); +} Status SchemaHelper::describe_table(const std::string& ip, const int32_t port, const TDescribeTableParams& request, diff --git a/be/src/exec/schema_scanner/schema_helper.h b/be/src/exec/schema_scanner/schema_helper.h index 72b7a9acf056c1..1aad5a202043d8 100644 --- a/be/src/exec/schema_scanner/schema_helper.h +++ b/be/src/exec/schema_scanner/schema_helper.h @@ -50,6 +50,9 @@ class SchemaHelper { static Status list_table_status(const std::string& ip, const int32_t port, const TGetTablesParams& table_params, TListTableStatusResult* table_result); + static Status list_simple_table_status(const std::string& ip, const int32_t port, + const TGetTablesParams& table_params, + TListSimpleTableStatusResult* table_result); static Status describe_table(const std::string& ip, const int32_t port, const TDescribeTableParams& desc_params, diff --git a/be/src/exec/schema_scanner/schema_simple_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_simple_tables_scanner.cpp new file mode 100644 index 00000000000000..33b35592a70c88 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_simple_tables_scanner.cpp @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/schema_scanner/schema_simple_tables_scanner.h" + +#include +#include +#include + +#include + +#include "common/status.h" +#include "exec/schema_scanner/schema_helper.h" +#include "runtime/decimalv2_value.h" +#include "runtime/define_primitive_type.h" +#include "util/runtime_profile.h" +#include "util/timezone_utils.h" +#include "vec/common/string_ref.h" +#include "vec/runtime/vdatetime_value.h" + +namespace doris { +class RuntimeState; + +namespace vectorized { +class Block; +} // namespace vectorized + +std::vector SchemaSimpleTablesScanner::_s_tbls_columns = { + // name, type, size, is_null + {"CATALOG_ID", TYPE_BIGINT, sizeof(int64_t), false}, + {"CATALOG_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, + {"DATABASE_ID", TYPE_BIGINT, sizeof(int64_t), false}, + {"DATABASE_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, + {"TABLE_ID", TYPE_BIGINT, sizeof(int64_t), true}, + {"TABLE_NAME", TYPE_VARCHAR, sizeof(StringRef), true}, +}; + +SchemaSimpleTablesScanner::SchemaSimpleTablesScanner() + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_SIMPLE_TABLES), _db_index(0) {} + +SchemaSimpleTablesScanner::~SchemaSimpleTablesScanner() {} + +Status SchemaSimpleTablesScanner::start(RuntimeState* state) { + if (!_is_init) { + return Status::InternalError("used before initialized."); + } + SCOPED_TIMER(_get_db_timer); + TGetDbsParams db_params; + if (nullptr != _param->db) { + db_params.__set_pattern(*(_param->db)); + } + if (nullptr != _param->catalog) { + db_params.__set_catalog(*(_param->catalog)); + } + if (nullptr != _param->current_user_ident) { + db_params.__set_current_user_ident(*(_param->current_user_ident)); + } else { + if (nullptr != _param->user) { + db_params.__set_user(*(_param->user)); + } + if (nullptr != _param->user_ip) { + db_params.__set_user_ip(*(_param->user_ip)); + } + } + + if (nullptr != _param->ip && 0 != _param->port) { + RETURN_IF_ERROR( + SchemaHelper::get_db_names(*(_param->ip), _param->port, db_params, &_db_result)); + } else { + return Status::InternalError("IP or port doesn't exists"); + } + return Status::OK(); +} + +Status SchemaSimpleTablesScanner::_get_new_table() { + SCOPED_TIMER(_get_table_timer); + TGetTablesParams table_params; + table_params.__set_db(_db_result.dbs[_db_index]); + if (_db_result.__isset.catalogs) { + table_params.__set_catalog(_db_result.catalogs[_db_index]); + } + _db_index++; + if (nullptr != _param->wild) { + table_params.__set_pattern(*(_param->wild)); + } + if (nullptr != _param->current_user_ident) { + table_params.__set_current_user_ident(*(_param->current_user_ident)); + } else { + if (nullptr != _param->user) { + table_params.__set_user(*(_param->user)); + } + if (nullptr != _param->user_ip) { + table_params.__set_user_ip(*(_param->user_ip)); + } + } + + if (nullptr != _param->ip && 0 != _param->port) { + RETURN_IF_ERROR(SchemaHelper::list_simple_table_status(*(_param->ip), _param->port, + table_params, &_table_result)); + } else { + return Status::InternalError("IP or port doesn't exists"); + } + return Status::OK(); +} + +Status SchemaSimpleTablesScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + auto table_num = _table_result.tables.size(); + if (table_num == 0) { + return Status::OK(); + } + std::vector null_datas(table_num, nullptr); + std::vector datas(table_num); + + // catalog_id + { + int64_t srcs[table_num]; + if (_db_result.__isset.catalog_ids) { + int64_t id = _db_result.catalog_ids[_db_index - 1]; + for (int i = 0; i < table_num; ++i) { + srcs[i] = id; + datas[i] = srcs + i; + } + fill_dest_column_for_range(block, 0, datas); + } else { + fill_dest_column_for_range(block, 0, null_datas); + } + } + + // catalog_name + { + if (_db_result.__isset.catalogs) { + std::string catalog_name = _db_result.catalogs[_db_index - 1]; + StringRef str_slot = StringRef(catalog_name.c_str(), catalog_name.size()); + for (int i = 0; i < table_num; ++i) { + datas[i] = &str_slot; + } + fill_dest_column_for_range(block, 1, datas); + } else { + fill_dest_column_for_range(block, 1, null_datas); + } + } + + // database_id + { + int64_t srcs[table_num]; + if (_db_result.__isset.db_ids) { + int64_t id = _db_result.db_ids[_db_index - 1]; + for (int i = 0; i < table_num; ++i) { + srcs[i] = id; + datas[i] = srcs + i; + } + fill_dest_column_for_range(block, 2, datas); + } else { + fill_dest_column_for_range(block, 2, null_datas); + } + } + + // database_name + { + if (_db_result.__isset.dbs) { + std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]); + StringRef str_slot = StringRef(db_name.c_str(), db_name.size()); + for (int i = 0; i < table_num; ++i) { + datas[i] = &str_slot; + } + fill_dest_column_for_range(block, 3, datas); + } else { + fill_dest_column_for_range(block, 3, null_datas); + } + } + // table_id + { + int64_t srcs[table_num]; + for (int i = 0; i < table_num; ++i) { + srcs[i] = _table_result.tables[i].id; + datas[i] = &srcs; + } + fill_dest_column_for_range(block, 4, datas); + } + + //table_name + { + StringRef strs[table_num]; + for (int i = 0; i < table_num; ++i) { + const std::string* src = &_table_result.tables[i].name; + strs[i] = StringRef(src->c_str(), src->size()); + datas[i] = strs + i; + } + fill_dest_column_for_range(block, 5, datas); + } + + return Status::OK(); +} + +Status SchemaSimpleTablesScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + if (_db_index < _db_result.dbs.size()) { + RETURN_IF_ERROR(_get_new_table()); + } else { + *eos = true; + return Status::OK(); + } + *eos = false; + return _fill_block_impl(block); +} + +} // namespace doris diff --git a/be/src/exec/schema_scanner/schema_simple_tables_scanner.h b/be/src/exec/schema_scanner/schema_simple_tables_scanner.h new file mode 100644 index 00000000000000..471051eb978f99 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_simple_tables_scanner.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "common/status.h" +#include "exec/schema_scanner.h" + +namespace doris { +class RuntimeState; + +namespace vectorized { +class Block; +} // namespace vectorized + +class SchemaSimpleTablesScanner : public SchemaScanner { + ENABLE_FACTORY_CREATOR(SchemaSimpleTablesScanner); + +public: + SchemaSimpleTablesScanner(); + ~SchemaSimpleTablesScanner() override; + + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; + +private: + Status _get_new_table(); + Status _fill_block_impl(vectorized::Block* block); + + int _db_index; + TGetDbsResult _db_result; + TListSimpleTableStatusResult _table_result; + static std::vector _s_tbls_columns; +}; + +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java index f1e6031d23dd0b..f506a9dce1d594 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java @@ -68,8 +68,8 @@ public enum SchemaTableType { SCH_CREATE_TABLE("CREATE_TABLE", "CREATE_TABLE", TSchemaTableType.SCH_CREATE_TABLE), SCH_INVALID("NULL", "NULL", TSchemaTableType.SCH_INVALID), SCH_ROWSETS("ROWSETS", "ROWSETS", TSchemaTableType.SCH_ROWSETS), - SCH_PARAMETERS("PARAMETERS", "PARAMETERS", TSchemaTableType.SCH_PARAMETERS); - + SCH_PARAMETERS("PARAMETERS", "PARAMETERS", TSchemaTableType.SCH_PARAMETERS), + SCH_SIMPLE_TABLES("SIMPLE_TABLES", "SIMPLE_TABLES", TSchemaTableType.SCH_SIMPLE_TABLES); private static final String dbName = "INFORMATION_SCHEMA"; private static SelectList fullSelectLists; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index d5035dbd52d3b9..a55f3f2046871d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -250,10 +250,10 @@ public class SchemaTable extends Table { // Compatible with mysql for mysqldump .put("column_statistics", new SchemaTable(SystemIdGenerator.getNextId(), "column_statistics", TableType.SCHEMA, - builder().column("SCHEMA_NAME", ScalarType.createVarchar(64)) - .column("TABLE_NAME", ScalarType.createVarchar(64)) - .column("COLUMN_NAME", ScalarType.createVarchar(64)) - .column("HISTOGRAM", ScalarType.createJsonbType()).build())) + builder().column("SCHEMA_NAME", ScalarType.createVarchar(64)) + .column("TABLE_NAME", ScalarType.createVarchar(64)) + .column("COLUMN_NAME", ScalarType.createVarchar(64)) + .column("HISTOGRAM", ScalarType.createJsonbType()).build())) .put("files", new SchemaTable(SystemIdGenerator.getNextId(), "files", TableType.SCHEMA, builder().column("FILE_ID", ScalarType.createType(PrimitiveType.BIGINT)) @@ -384,37 +384,45 @@ public class SchemaTable extends Table { .column("COLLATION_CONNECTION", ScalarType.createVarchar(32)) .column("DATABASE_COLLATION", ScalarType.createVarchar(32)).build())) .put("rowsets", new SchemaTable(SystemIdGenerator.getNextId(), "rowsets", TableType.SCHEMA, - builder().column("BACKEND_ID", ScalarType.createType(PrimitiveType.BIGINT)) - .column("ROWSET_ID", ScalarType.createVarchar(64)) - .column("TABLET_ID", ScalarType.createType(PrimitiveType.BIGINT)) - .column("ROWSET_NUM_ROWS", ScalarType.createType(PrimitiveType.BIGINT)) - .column("TXN_ID", ScalarType.createType(PrimitiveType.BIGINT)) - .column("NUM_SEGMENTS", ScalarType.createType(PrimitiveType.BIGINT)) - .column("START_VERSION", ScalarType.createType(PrimitiveType.BIGINT)) - .column("END_VERSION", ScalarType.createType(PrimitiveType.BIGINT)) - .column("INDEX_DISK_SIZE", ScalarType.createType(PrimitiveType.BIGINT)) - .column("DATA_DISK_SIZE", ScalarType.createType(PrimitiveType.BIGINT)) - .column("CREATION_TIME", ScalarType.createType(PrimitiveType.BIGINT)) - .column("NEWEST_WRITE_TIMESTAMP", ScalarType.createType(PrimitiveType.BIGINT)) - .build())) + builder().column("BACKEND_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("ROWSET_ID", ScalarType.createVarchar(64)) + .column("TABLET_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("ROWSET_NUM_ROWS", ScalarType.createType(PrimitiveType.BIGINT)) + .column("TXN_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("NUM_SEGMENTS", ScalarType.createType(PrimitiveType.BIGINT)) + .column("START_VERSION", ScalarType.createType(PrimitiveType.BIGINT)) + .column("END_VERSION", ScalarType.createType(PrimitiveType.BIGINT)) + .column("INDEX_DISK_SIZE", ScalarType.createType(PrimitiveType.BIGINT)) + .column("DATA_DISK_SIZE", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CREATION_TIME", ScalarType.createType(PrimitiveType.BIGINT)) + .column("NEWEST_WRITE_TIMESTAMP", ScalarType.createType(PrimitiveType.BIGINT)) + .build())) .put("parameters", new SchemaTable(SystemIdGenerator.getNextId(), "parameters", TableType.SCHEMA, - builder().column("SPECIFIC_CATALOG", ScalarType.createVarchar(64)) - .column("SPECIFIC_SCHEMA", ScalarType.createVarchar(64)) - .column("SPECIFIC_NAME", ScalarType.createVarchar(64)) - .column("ORDINAL_POSITION", ScalarType.createVarchar(77)) - .column("PARAMETER_MODE", ScalarType.createVarchar(77)) - .column("PARAMETER_NAME", ScalarType.createVarchar(77)) - .column("DATA_TYPE", ScalarType.createVarchar(64)) - .column("CHARACTER_OCTET_LENGTH", ScalarType.createVarchar(64)) - .column("NUMERIC_PRECISION", ScalarType.createVarchar(512)) - .column("NUMERIC_SCALE", ScalarType.createVarchar(64)) - .column("DATETIME_PRECISION", ScalarType.createVarchar(64)) - .column("CHARACTER_SET_NAME", ScalarType.createVarchar(256)) - .column("COLLATION_NAME", ScalarType.createVarchar(64)) - .column("DTD_IDENTIFIER", ScalarType.createVarchar(64)) - .column("ROUTINE_TYPE", ScalarType.createVarchar(64)) - .column("DATA_TYPEDTD_IDENDS", ScalarType.createVarchar(64)) - .build())) + builder().column("SPECIFIC_CATALOG", ScalarType.createVarchar(64)) + .column("SPECIFIC_SCHEMA", ScalarType.createVarchar(64)) + .column("SPECIFIC_NAME", ScalarType.createVarchar(64)) + .column("ORDINAL_POSITION", ScalarType.createVarchar(77)) + .column("PARAMETER_MODE", ScalarType.createVarchar(77)) + .column("PARAMETER_NAME", ScalarType.createVarchar(77)) + .column("DATA_TYPE", ScalarType.createVarchar(64)) + .column("CHARACTER_OCTET_LENGTH", ScalarType.createVarchar(64)) + .column("NUMERIC_PRECISION", ScalarType.createVarchar(512)) + .column("NUMERIC_SCALE", ScalarType.createVarchar(64)) + .column("DATETIME_PRECISION", ScalarType.createVarchar(64)) + .column("CHARACTER_SET_NAME", ScalarType.createVarchar(256)) + .column("COLLATION_NAME", ScalarType.createVarchar(64)) + .column("DTD_IDENTIFIER", ScalarType.createVarchar(64)) + .column("ROUTINE_TYPE", ScalarType.createVarchar(64)) + .column("DATA_TYPEDTD_IDENDS", ScalarType.createVarchar(64)) + .build())) + .put("simple_tables", new SchemaTable(SystemIdGenerator.getNextId(), "simple_tables", TableType.SCHEMA, + builder().column("CATALOG_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CATALOG_NAME", ScalarType.createVarchar(FN_REFLEN)) + .column("DATABASE_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("DATABASE_NAME", ScalarType.createVarchar(NAME_CHAR_LEN)) + .column("TABLE_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("TABLE_NAME", ScalarType.createVarchar(NAME_CHAR_LEN)) + .build())) .build(); protected SchemaTable(long id, String name, TableType type, List baseSchema) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 9dbc40ff8476ca..f14a0db009bc5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -582,6 +582,7 @@ public boolean useSelfSplitter() { @Override public Collection getAllDbs() { + makeSureInitialized(); return new HashSet<>(idToDb.values()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index a61b49781a8e57..140600d39c1857 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -66,6 +66,8 @@ public class SessionVariable implements Serializable, Writable { public static final String EXEC_MEM_LIMIT = "exec_mem_limit"; public static final String SCAN_QUEUE_MEM_LIMIT = "scan_queue_mem_limit"; public static final String QUERY_TIMEOUT = "query_timeout"; + + public static final String QUERY_SIMPLE_TABLES_TIMEOUT = "query_simple_tables_timeout"; public static final String MAX_EXECUTION_TIME = "max_execution_time"; public static final String INSERT_TIMEOUT = "insert_timeout"; public static final String ENABLE_PROFILE = "enable_profile"; @@ -439,6 +441,10 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = INSERT_TIMEOUT) public int insertTimeoutS = 14400; + //query internal.information_schema.simple_tables time + @VariableMgr.VarAttr(name = QUERY_SIMPLE_TABLES_TIMEOUT) + public int querySimpleTablesTimeout = 3; + // if true, need report to coordinator when plan fragment execute successfully. @VariableMgr.VarAttr(name = ENABLE_PROFILE, needForward = true) public boolean enableProfile = false; @@ -1305,6 +1311,10 @@ public int getInsertTimeoutS() { return insertTimeoutS; } + public int getQuerySimpleTablesTimeout() { + return querySimpleTablesTimeout; + } + public void setInsertTimeoutS(int insertTimeoutS) { this.insertTimeoutS = insertTimeoutS; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 27649ef659445e..842e14ef34ed96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -145,6 +145,7 @@ import org.apache.doris.thrift.TInitExternalCtlMetaRequest; import org.apache.doris.thrift.TInitExternalCtlMetaResult; import org.apache.doris.thrift.TListPrivilegesResult; +import org.apache.doris.thrift.TListSimpleTableStatusResult; import org.apache.doris.thrift.TListTableStatusResult; import org.apache.doris.thrift.TLoadTxn2PCRequest; import org.apache.doris.thrift.TLoadTxn2PCResult; @@ -177,6 +178,7 @@ import org.apache.doris.thrift.TRollbackTxnResult; import org.apache.doris.thrift.TShowVariableRequest; import org.apache.doris.thrift.TShowVariableResult; +import org.apache.doris.thrift.TSimpleTableStatus; import org.apache.doris.thrift.TSnapshotLoaderReportRequest; import org.apache.doris.thrift.TSnapshotType; import org.apache.doris.thrift.TStatus; @@ -216,6 +218,7 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; @@ -332,8 +335,11 @@ public TGetDbsResult getDbNames(TGetDbsParams params) throws TException { LOG.debug("get db request: {}", params); TGetDbsResult result = new TGetDbsResult(); - List dbs = Lists.newArrayList(); - List catalogs = Lists.newArrayList(); + List dbNames = Lists.newArrayList(); + List catalogNames = Lists.newArrayList(); + List dbIds = Lists.newArrayList(); + List catalogIds = Lists.newArrayList(); + PatternMatcher matcher = null; if (params.isSetPattern()) { try { @@ -353,16 +359,16 @@ public TGetDbsResult getDbNames(TGetDbsParams params) throws TException { .getCatalogOrException(params.catalog, catalog -> new TException("Unknown catalog " + catalog))); } for (CatalogIf catalog : catalogIfs) { - List dbNames; + Collection dbs; try { - dbNames = catalog.getDbNamesOrEmpty(); + dbs = catalog.getAllDbs(); } catch (Exception e) { LOG.warn("failed to get database names for catalog {}", catalog.getName(), e); // Some external catalog may fail to get databases due to wrong connection info. // So continue here to get databases of other catalogs. continue; } - LOG.debug("get db names: {}, in catalog: {}", dbNames, catalog.getName()); + LOG.debug("get db size: {}, in catalog: {}", dbs.size(), catalog.getName()); UserIdentity currentUser = null; if (params.isSetCurrentUserIdent()) { @@ -370,23 +376,27 @@ public TGetDbsResult getDbNames(TGetDbsParams params) throws TException { } else { currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip); } - for (String fullName : dbNames) { + for (DatabaseIf db : dbs) { + String fullName = db.getFullName(); if (!env.getAccessManager().checkDbPriv(currentUser, fullName, PrivPredicate.SHOW)) { continue; } - final String db = ClusterNamespace.getNameFromFullName(fullName); - if (matcher != null && !matcher.match(db)) { + if (matcher != null && !matcher.match(ClusterNamespace.getNameFromFullName(fullName))) { continue; } - catalogs.add(catalog.getName()); - dbs.add(fullName); + catalogNames.add(catalog.getName()); + dbNames.add(fullName); + catalogIds.add(catalog.getId()); + dbIds.add(db.getId()); } } - result.setDbs(dbs); - result.setCatalogs(catalogs); + result.setDbs(dbNames); + result.setCatalogs(catalogNames); + result.setCatalogIds(catalogIds); + result.setDbIds(dbIds); return result; } @@ -687,6 +697,74 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE return result; } + public TListSimpleTableStatusResult listSimpleTableStatus(TGetTablesParams params) throws TException { + + long timeoutTs = System.currentTimeMillis() + exeEnv.getScheduler().getContext(0) + .getSessionVariable().getQuerySimpleTablesTimeout() * 1000L; + + LOG.debug("get list simple table request: {}", params); + TListSimpleTableStatusResult result = new TListSimpleTableStatusResult(); + List tablesResult = Lists.newArrayList(); + result.setTables(tablesResult); + PatternMatcher matcher = null; + if (params.isSetPattern()) { + try { + matcher = PatternMatcher.createMysqlPattern(params.getPattern(), + CaseSensibility.TABLE.getCaseSensibility()); + } catch (PatternMatcherException e) { + throw new TException("Pattern is in bad format " + params.getPattern()); + } + } + + UserIdentity currentUser; + if (params.isSetCurrentUserIdent()) { + currentUser = UserIdentity.fromThrift(params.current_user_ident); + } else { + currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip); + } + + String catalogName = InternalCatalog.INTERNAL_CATALOG_NAME; + if (params.isSetCatalog()) { + catalogName = params.catalog; + } + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName); + if (catalog != null) { + DatabaseIf db = catalog.getDbNullable(params.db); + if (db != null) { + try { + List tables = db.getTables(); + for (TableIf table : tables) { + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(currentUser, params.db, + table.getName(), PrivPredicate.SHOW)) { + continue; + } + table.readLock(); + try { + if (matcher != null && !matcher.match(table.getName())) { + continue; + } + TSimpleTableStatus status = new TSimpleTableStatus(); + status.setName(table.getName()); + status.setId(table.getId()); + + tablesResult.add(status); + } finally { + table.readUnlock(); + } + + long currentTs = System.currentTimeMillis(); + if (currentTs >= timeoutTs) { //limit query time + break; + } + } + } catch (Exception e) { + LOG.warn("failed to get tables for db {} in catalog {}", db.getFullName(), catalogName, e); + } + } + } + return result; + } + @Override public TListPrivilegesResult listTablePrivilegeStatus(TGetTablesParams params) throws TException { LOG.debug("get list table privileges request: {}", params); diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index cb64ddb70dd4f9..0f6dcb442f20b2 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -117,7 +117,8 @@ enum TSchemaTableType { SCH_ROWSETS, SCH_BACKENDS, SCH_COLUMN_STATISTICS, - SCH_PARAMETERS; + SCH_PARAMETERS, + SCH_SIMPLE_TABLES; } enum THdfsCompression { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index d916cfe9c1c53f..1e16b657d8630b 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -310,10 +310,12 @@ struct TGetDbsParams { 5: optional string catalog } -// getDbNames returns a list of database names and catalog names +// getDbNames returns a list of database names , database ids and catalog names ,catalog ids struct TGetDbsResult { 1: optional list dbs 2: optional list catalogs + 3: optional list db_ids + 4: optional list catalog_ids } // Arguments to getTableNames, which returns a list of tables that match an @@ -351,6 +353,14 @@ struct TListTableStatusResult { 1: required list tables } +struct TSimpleTableStatus { + 1: required string name + 2: required i64 id +} +struct TListSimpleTableStatusResult { + 1: required list tables +} + // getTableNames returns a list of unqualified table names struct TGetTablesResult { 1: list tables @@ -1134,6 +1144,7 @@ service FrontendService { TMasterOpResult forward(1: TMasterOpRequest params) TListTableStatusResult listTableStatus(1: TGetTablesParams params) + TListSimpleTableStatusResult listSimpleTableStatus(1: TGetTablesParams params) TListPrivilegesResult listTablePrivilegeStatus(1: TGetTablesParams params) TListPrivilegesResult listSchemaPrivilegeStatus(1: TGetTablesParams params) TListPrivilegesResult listUserPrivilegeStatus(1: TGetTablesParams params) From d32355d6aae7a776cf987c0008749becfe424899 Mon Sep 17 00:00:00 2001 From: changyuwei <2017501503@qq.com> Date: Tue, 8 Aug 2023 12:04:02 +0800 Subject: [PATCH 2/8] add information_schema.profiling in order to Compatible with mysql --- be/src/exec/schema_scanner.cpp | 3 + .../schema_profiling_scanner.cpp | 102 ++++++++++++++++++ .../schema_scanner/schema_profiling_scanner.h | 46 ++++++++ .../doris/analysis/SchemaTableType.java | 3 +- .../org/apache/doris/catalog/SchemaTable.java | 20 ++++ .../org/apache/doris/qe/SessionVariable.java | 5 + gensrc/thrift/Descriptors.thrift | 3 +- 7 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 be/src/exec/schema_scanner/schema_profiling_scanner.cpp create mode 100644 be/src/exec/schema_scanner/schema_profiling_scanner.h diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 9355bc1b194f3c..09edce08c18dfb 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -32,6 +32,7 @@ #include "exec/schema_scanner/schema_dummy_scanner.h" #include "exec/schema_scanner/schema_files_scanner.h" #include "exec/schema_scanner/schema_partitions_scanner.h" +#include "exec/schema_scanner/schema_profiling_scanner.h" #include "exec/schema_scanner/schema_rowsets_scanner.h" #include "exec/schema_scanner/schema_schema_privileges_scanner.h" #include "exec/schema_scanner/schema_schemata_scanner.h" @@ -147,6 +148,8 @@ std::unique_ptr SchemaScanner::create(TSchemaTableType::type type return SchemaRowsetsScanner::create_unique(); case TSchemaTableType::SCH_SIMPLE_TABLES: return SchemaSimpleTablesScanner::create_unique(); + case TSchemaTableType::SCH_PROFILING: + return SchemaProfilingScanner::create_unique(); default: return SchemaDummyScanner::create_unique(); break; diff --git a/be/src/exec/schema_scanner/schema_profiling_scanner.cpp b/be/src/exec/schema_scanner/schema_profiling_scanner.cpp new file mode 100644 index 00000000000000..d2bd8b256fced1 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_profiling_scanner.cpp @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/schema_scanner/schema_profiling_scanner.h" + +#include +#include +#include + +#include "exec/schema_scanner/schema_helper.h" +#include "runtime/define_primitive_type.h" +#include "util/runtime_profile.h" +#include "vec/common/string_ref.h" + +namespace doris { +class RuntimeState; +namespace vectorized { +class Block; +} // namespace vectorized + +std::vector SchemaProfilingScanner::_s_tbls_columns = { + // name, type, size, is_null + {"QUERY_ID", TYPE_INT, sizeof(int), false}, + {"SEQ", TYPE_INT, sizeof(int), false}, + {"STATE", TYPE_VARCHAR, sizeof(StringRef), false}, + {"DURATION", TYPE_DOUBLE, sizeof(double), false}, + {"CPU_USER", TYPE_DOUBLE, sizeof(double), true}, + {"CPU_SYSTEM", TYPE_DOUBLE, sizeof(double), true}, + {"CONTEXT_VOLUNTARY", TYPE_INT, sizeof(int), true}, + {"CONTEXT_INVOLUNTARY", TYPE_INT, sizeof(int), true}, + {"BLOCK_OPS_IN", TYPE_INT, sizeof(int), true}, + {"BLOCK_OPS_OUT", TYPE_INT, sizeof(int), true}, + {"MESSAGES_SENT", TYPE_INT, sizeof(int), true}, + {"MESSAGES_RECEIVED", TYPE_INT, sizeof(int), true}, + {"PAGE_FAULTS_MAJOR", TYPE_INT, sizeof(int), true}, + {"PAGE_FAULTS_MINOR", TYPE_INT, sizeof(int), true}, + {"SWAPS", TYPE_INT, sizeof(int), true}, + {"SOURCE_FUNCTION", TYPE_VARCHAR, sizeof(StringRef), false}, + {"SOURCE_FILE", TYPE_VARCHAR, sizeof(StringRef), false}, + {"SOURCE_LINE", TYPE_INT, sizeof(int), true}, +}; + +SchemaProfilingScanner::SchemaProfilingScanner() + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_PROFILING) {} + +SchemaProfilingScanner::~SchemaProfilingScanner() {} + +Status SchemaProfilingScanner::start(RuntimeState* state) { + if (!_is_init) { + return Status::InternalError("used before initialized."); + } + SCOPED_TIMER(_get_db_timer); + TGetDbsParams db_params; + if (nullptr != _param->db) { + db_params.__set_pattern(*(_param->db)); + } + if (nullptr != _param->catalog) { + db_params.__set_catalog(*(_param->catalog)); + } + if (nullptr != _param->current_user_ident) { + db_params.__set_current_user_ident(*(_param->current_user_ident)); + } else { + if (nullptr != _param->user) { + db_params.__set_user(*(_param->user)); + } + if (nullptr != _param->user_ip) { + db_params.__set_user_ip(*(_param->user_ip)); + } + } + + if (nullptr == _param->ip || 0 == _param->port) { + return Status::InternalError("IP or port doesn't exists"); + } + return Status::OK(); +} + +Status SchemaProfilingScanner::get_next_block(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + *eos = true; + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/exec/schema_scanner/schema_profiling_scanner.h b/be/src/exec/schema_scanner/schema_profiling_scanner.h new file mode 100644 index 00000000000000..5399cb14eb43f5 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_profiling_scanner.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +#include "common/status.h" +#include "exec/schema_scanner.h" + +namespace doris { +class RuntimeState; +namespace vectorized { +class Block; +} // namespace vectorized + +class SchemaProfilingScanner : public SchemaScanner { + ENABLE_FACTORY_CREATOR(SchemaProfilingScanner); + +public: + SchemaProfilingScanner(); + ~SchemaProfilingScanner() override; + + Status start(RuntimeState* state) override; + Status get_next_block(vectorized::Block* block, bool* eos) override; + + static std::vector _s_tbls_columns; +}; + +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java index f506a9dce1d594..e58bc24cde9133 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java @@ -69,7 +69,8 @@ public enum SchemaTableType { SCH_INVALID("NULL", "NULL", TSchemaTableType.SCH_INVALID), SCH_ROWSETS("ROWSETS", "ROWSETS", TSchemaTableType.SCH_ROWSETS), SCH_PARAMETERS("PARAMETERS", "PARAMETERS", TSchemaTableType.SCH_PARAMETERS), - SCH_SIMPLE_TABLES("SIMPLE_TABLES", "SIMPLE_TABLES", TSchemaTableType.SCH_SIMPLE_TABLES); + SCH_SIMPLE_TABLES("SIMPLE_TABLES", "SIMPLE_TABLES", TSchemaTableType.SCH_SIMPLE_TABLES), + SCH_PROFILING("PROFILING", "PROFILING", TSchemaTableType.SCH_PROFILING); private static final String dbName = "INFORMATION_SCHEMA"; private static SelectList fullSelectLists; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index a55f3f2046871d..dae421e9ff23a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -423,6 +423,26 @@ public class SchemaTable extends Table { .column("TABLE_ID", ScalarType.createType(PrimitiveType.BIGINT)) .column("TABLE_NAME", ScalarType.createVarchar(NAME_CHAR_LEN)) .build())) + .put("profiling", new SchemaTable(SystemIdGenerator.getNextId(), "profiling", TableType.SCHEMA, + builder().column("QUERY_ID", ScalarType.createType(PrimitiveType.INT)) + .column("SEQ", ScalarType.createType(PrimitiveType.INT)) + .column("STATE", ScalarType.createVarchar(30)) + .column("DURATION", ScalarType.createType(PrimitiveType.DOUBLE)) + .column("CPU_USER", ScalarType.createType(PrimitiveType.DOUBLE)) + .column("CPU_SYSTEM", ScalarType.createType(PrimitiveType.DOUBLE)) + .column("CONTEXT_VOLUNTARY", ScalarType.createType(PrimitiveType.INT)) + .column("CONTEXT_INVOLUNTARY", ScalarType.createType(PrimitiveType.INT)) + .column("BLOCK_OPS_IN", ScalarType.createType(PrimitiveType.INT)) + .column("BLOCK_OPS_OUT", ScalarType.createType(PrimitiveType.INT)) + .column("MESSAGES_SENT", ScalarType.createType(PrimitiveType.INT)) + .column("MESSAGES_RECEIVED", ScalarType.createType(PrimitiveType.INT)) + .column("PAGE_FAULTS_MAJOR", ScalarType.createType(PrimitiveType.INT)) + .column("PAGE_FAULTS_MINOR", ScalarType.createType(PrimitiveType.INT)) + .column("SWAPS", ScalarType.createType(PrimitiveType.INT)) + .column("SOURCE_FUNCTION", ScalarType.createVarchar(30)) + .column("SOURCE_FILE", ScalarType.createVarchar(20)) + .column("SOURCE_LINE", ScalarType.createType(PrimitiveType.INT)) + .build())) .build(); protected SchemaTable(long id, String name, TableType type, List baseSchema) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 140600d39c1857..1e011d702f3529 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -135,6 +135,9 @@ public class SessionVariable implements Serializable, Writable { public static final String DEFAULT_STORAGE_ENGINE = "default_storage_engine"; public static final String DEFAULT_TMP_STORAGE_ENGINE = "default_tmp_storage_engine"; + // Compatible with mysql + public static final String PROFILLING = "profiling"; + public static final String DIV_PRECISION_INCREMENT = "div_precision_increment"; // see comment of `doris_max_scan_key_num` and `max_pushdown_conditions_per_column` in BE config @@ -726,6 +729,8 @@ public class SessionVariable implements Serializable, Writable { public int getBeNumberForTest() { return beNumberForTest; } + @VariableMgr.VarAttr(name = PROFILLING) + public boolean profiling = false; public void setBeNumberForTest(int beNumberForTest) { this.beNumberForTest = beNumberForTest; diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 0f6dcb442f20b2..ec9269ac7a2070 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -118,7 +118,8 @@ enum TSchemaTableType { SCH_BACKENDS, SCH_COLUMN_STATISTICS, SCH_PARAMETERS, - SCH_SIMPLE_TABLES; + SCH_SIMPLE_TABLES, + SCH_PROFILING; } enum THdfsCompression { From 9381753b135be6fc95f1e61de9b74458981ad42b Mon Sep 17 00:00:00 2001 From: changyuwei <2017501503@qq.com> Date: Fri, 11 Aug 2023 13:39:52 +0800 Subject: [PATCH 3/8] [feature](information_schema)add information_schema.simple_tables for quickly get catlogs,db,table. --- be/src/exec/schema_scanner.cpp | 6 +-- be/src/exec/schema_scanner/schema_helper.cpp | 8 +-- be/src/exec/schema_scanner/schema_helper.h | 6 +-- ...p => schema_metadata_name_ids_scanner.cpp} | 50 +++++++++++-------- ...r.h => schema_metadata_name_ids_scanner.h} | 10 ++-- .../java/org/apache/doris/common/Config.java | 7 +++ .../doris/analysis/SchemaTableType.java | 2 +- .../org/apache/doris/catalog/SchemaTable.java | 3 +- .../org/apache/doris/qe/SessionVariable.java | 9 +--- .../doris/service/FrontendServiceImpl.java | 16 +++--- gensrc/thrift/Descriptors.thrift | 4 +- gensrc/thrift/FrontendService.thrift | 13 ++--- 12 files changed, 72 insertions(+), 62 deletions(-) rename be/src/exec/schema_scanner/{schema_simple_tables_scanner.cpp => schema_metadata_name_ids_scanner.cpp} (79%) rename be/src/exec/schema_scanner/{schema_simple_tables_scanner.h => schema_metadata_name_ids_scanner.h} (84%) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 09edce08c18dfb..9733558284a8fd 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -31,12 +31,12 @@ #include "exec/schema_scanner/schema_columns_scanner.h" #include "exec/schema_scanner/schema_dummy_scanner.h" #include "exec/schema_scanner/schema_files_scanner.h" +#include "exec/schema_scanner/schema_metadata_name_ids_scanner.h" #include "exec/schema_scanner/schema_partitions_scanner.h" #include "exec/schema_scanner/schema_profiling_scanner.h" #include "exec/schema_scanner/schema_rowsets_scanner.h" #include "exec/schema_scanner/schema_schema_privileges_scanner.h" #include "exec/schema_scanner/schema_schemata_scanner.h" -#include "exec/schema_scanner/schema_simple_tables_scanner.h" #include "exec/schema_scanner/schema_table_privileges_scanner.h" #include "exec/schema_scanner/schema_tables_scanner.h" #include "exec/schema_scanner/schema_user_privileges_scanner.h" @@ -146,8 +146,8 @@ std::unique_ptr SchemaScanner::create(TSchemaTableType::type type return SchemaPartitionsScanner::create_unique(); case TSchemaTableType::SCH_ROWSETS: return SchemaRowsetsScanner::create_unique(); - case TSchemaTableType::SCH_SIMPLE_TABLES: - return SchemaSimpleTablesScanner::create_unique(); + case TSchemaTableType::SCH_METADATA_NAME_IDS: + return SchemaMetadataNameIdsScanner::create_unique(); case TSchemaTableType::SCH_PROFILING: return SchemaProfilingScanner::create_unique(); default: diff --git a/be/src/exec/schema_scanner/schema_helper.cpp b/be/src/exec/schema_scanner/schema_helper.cpp index f811a2e628aa16..3184aed4d2e4f5 100644 --- a/be/src/exec/schema_scanner/schema_helper.cpp +++ b/be/src/exec/schema_scanner/schema_helper.cpp @@ -60,12 +60,12 @@ Status SchemaHelper::list_table_status(const std::string& ip, const int32_t port client->listTableStatus(*result, request); }); } -Status SchemaHelper::list_simple_table_status(const std::string& ip, const int32_t port, - const doris::TGetTablesParams& request, - TListSimpleTableStatusResult* result) { +Status SchemaHelper::list_table_metadata_name_ids(const std::string& ip, const int32_t port, + const doris::TGetTablesParams& request, + TListTableMetadataNameIdsResult* result) { return ThriftRpcHelper::rpc( ip, port, [&request, &result](FrontendServiceConnection& client) { - client->listSimpleTableStatus(*result, request); + client->listTableMetadataNameIds(*result, request); }); } diff --git a/be/src/exec/schema_scanner/schema_helper.h b/be/src/exec/schema_scanner/schema_helper.h index 1aad5a202043d8..590afb4ecf2e17 100644 --- a/be/src/exec/schema_scanner/schema_helper.h +++ b/be/src/exec/schema_scanner/schema_helper.h @@ -50,9 +50,9 @@ class SchemaHelper { static Status list_table_status(const std::string& ip, const int32_t port, const TGetTablesParams& table_params, TListTableStatusResult* table_result); - static Status list_simple_table_status(const std::string& ip, const int32_t port, - const TGetTablesParams& table_params, - TListSimpleTableStatusResult* table_result); + static Status list_table_metadata_name_ids(const std::string& ip, const int32_t port, + const doris::TGetTablesParams& request, + TListTableMetadataNameIdsResult* result); static Status describe_table(const std::string& ip, const int32_t port, const TDescribeTableParams& desc_params, diff --git a/be/src/exec/schema_scanner/schema_simple_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp similarity index 79% rename from be/src/exec/schema_scanner/schema_simple_tables_scanner.cpp rename to be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp index 33b35592a70c88..805fc52df69ca2 100644 --- a/be/src/exec/schema_scanner/schema_simple_tables_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "exec/schema_scanner/schema_simple_tables_scanner.h" +#include "exec/schema_scanner/schema_metadata_name_ids_scanner.h" #include #include @@ -39,22 +39,22 @@ namespace vectorized { class Block; } // namespace vectorized -std::vector SchemaSimpleTablesScanner::_s_tbls_columns = { +std::vector SchemaMetadataNameIdsScanner::_s_tbls_columns = { // name, type, size, is_null - {"CATALOG_ID", TYPE_BIGINT, sizeof(int64_t), false}, - {"CATALOG_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, - {"DATABASE_ID", TYPE_BIGINT, sizeof(int64_t), false}, - {"DATABASE_NAME", TYPE_VARCHAR, sizeof(StringRef), false}, + {"CATALOG_ID", TYPE_BIGINT, sizeof(int64_t), true}, + {"CATALOG_NAME", TYPE_VARCHAR, sizeof(StringRef), true}, + {"DATABASE_ID", TYPE_BIGINT, sizeof(int64_t), true}, + {"DATABASE_NAME", TYPE_VARCHAR, sizeof(StringRef), true}, {"TABLE_ID", TYPE_BIGINT, sizeof(int64_t), true}, {"TABLE_NAME", TYPE_VARCHAR, sizeof(StringRef), true}, }; -SchemaSimpleTablesScanner::SchemaSimpleTablesScanner() - : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_SIMPLE_TABLES), _db_index(0) {} +SchemaMetadataNameIdsScanner::SchemaMetadataNameIdsScanner() + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_METADATA_NAME_IDS), _db_index(0) {} -SchemaSimpleTablesScanner::~SchemaSimpleTablesScanner() {} +SchemaMetadataNameIdsScanner::~SchemaMetadataNameIdsScanner() {} -Status SchemaSimpleTablesScanner::start(RuntimeState* state) { +Status SchemaMetadataNameIdsScanner::start(RuntimeState* state) { if (!_is_init) { return Status::InternalError("used before initialized."); } @@ -86,7 +86,7 @@ Status SchemaSimpleTablesScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaSimpleTablesScanner::_get_new_table() { +Status SchemaMetadataNameIdsScanner::_get_new_table() { SCOPED_TIMER(_get_table_timer); TGetTablesParams table_params; table_params.__set_db(_db_result.dbs[_db_index]); @@ -109,15 +109,15 @@ Status SchemaSimpleTablesScanner::_get_new_table() { } if (nullptr != _param->ip && 0 != _param->port) { - RETURN_IF_ERROR(SchemaHelper::list_simple_table_status(*(_param->ip), _param->port, - table_params, &_table_result)); + RETURN_IF_ERROR(SchemaHelper::list_table_metadata_name_ids(*(_param->ip), _param->port, + table_params, &_table_result)); } else { return Status::InternalError("IP or port doesn't exists"); } return Status::OK(); } -Status SchemaSimpleTablesScanner::_fill_block_impl(vectorized::Block* block) { +Status SchemaMetadataNameIdsScanner::_fill_block_impl(vectorized::Block* block) { SCOPED_TIMER(_fill_block_timer); auto table_num = _table_result.tables.size(); if (table_num == 0) { @@ -183,12 +183,16 @@ Status SchemaSimpleTablesScanner::_fill_block_impl(vectorized::Block* block) { fill_dest_column_for_range(block, 3, null_datas); } } - // table_id + // table_id { int64_t srcs[table_num]; for (int i = 0; i < table_num; ++i) { - srcs[i] = _table_result.tables[i].id; - datas[i] = &srcs; + if (_table_result.tables[i].__isset.id) { + srcs[i] = _table_result.tables[i].id; + datas[i] = &srcs; + } else { + datas[i] = nullptr; + } } fill_dest_column_for_range(block, 4, datas); } @@ -197,9 +201,13 @@ Status SchemaSimpleTablesScanner::_fill_block_impl(vectorized::Block* block) { { StringRef strs[table_num]; for (int i = 0; i < table_num; ++i) { - const std::string* src = &_table_result.tables[i].name; - strs[i] = StringRef(src->c_str(), src->size()); - datas[i] = strs + i; + if (_table_result.tables[i].__isset.name) { + const std::string* src = &_table_result.tables[i].name; + strs[i] = StringRef(src->c_str(), src->size()); + datas[i] = strs + i; + } else { + datas[i] = nullptr; + } } fill_dest_column_for_range(block, 5, datas); } @@ -207,7 +215,7 @@ Status SchemaSimpleTablesScanner::_fill_block_impl(vectorized::Block* block) { return Status::OK(); } -Status SchemaSimpleTablesScanner::get_next_block(vectorized::Block* block, bool* eos) { +Status SchemaMetadataNameIdsScanner::get_next_block(vectorized::Block* block, bool* eos) { if (!_is_init) { return Status::InternalError("Used before initialized."); } diff --git a/be/src/exec/schema_scanner/schema_simple_tables_scanner.h b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.h similarity index 84% rename from be/src/exec/schema_scanner/schema_simple_tables_scanner.h rename to be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.h index 471051eb978f99..9981d441d856aa 100644 --- a/be/src/exec/schema_scanner/schema_simple_tables_scanner.h +++ b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.h @@ -31,12 +31,12 @@ namespace vectorized { class Block; } // namespace vectorized -class SchemaSimpleTablesScanner : public SchemaScanner { - ENABLE_FACTORY_CREATOR(SchemaSimpleTablesScanner); +class SchemaMetadataNameIdsScanner : public SchemaScanner { + ENABLE_FACTORY_CREATOR(SchemaMetadataNameIdsScanner); public: - SchemaSimpleTablesScanner(); - ~SchemaSimpleTablesScanner() override; + SchemaMetadataNameIdsScanner(); + ~SchemaMetadataNameIdsScanner() override; Status start(RuntimeState* state) override; Status get_next_block(vectorized::Block* block, bool* eos) override; @@ -47,7 +47,7 @@ class SchemaSimpleTablesScanner : public SchemaScanner { int _db_index; TGetDbsResult _db_result; - TListSimpleTableStatusResult _table_result; + TListTableMetadataNameIdsResult _table_result; static std::vector _s_tbls_columns; }; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 9da22b20da5932..fa1cb97acfc190 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2122,4 +2122,11 @@ public class Config extends ConfigBase { @ConfField public static int table_stats_health_threshold = 80; + @ConfField(mutable = true, masterOnly = false, description = { + "查询information_schema.metadata_name_ids表时,获取一个数据库中所有表用的时间", + "When querying the information_schema.metadata_name_ids table," + + " the time used to obtain all tables in one database" + }) + public static long query_metadata_name_ids_timeout = 3; + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java index e58bc24cde9133..10c958c47b0bb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java @@ -69,7 +69,7 @@ public enum SchemaTableType { SCH_INVALID("NULL", "NULL", TSchemaTableType.SCH_INVALID), SCH_ROWSETS("ROWSETS", "ROWSETS", TSchemaTableType.SCH_ROWSETS), SCH_PARAMETERS("PARAMETERS", "PARAMETERS", TSchemaTableType.SCH_PARAMETERS), - SCH_SIMPLE_TABLES("SIMPLE_TABLES", "SIMPLE_TABLES", TSchemaTableType.SCH_SIMPLE_TABLES), + SCH_METADATA_NAME_IDS("METADATA_NAME_IDS", "METADATA_NAME_IDS", TSchemaTableType.SCH_METADATA_NAME_IDS), SCH_PROFILING("PROFILING", "PROFILING", TSchemaTableType.SCH_PROFILING); private static final String dbName = "INFORMATION_SCHEMA"; private static SelectList fullSelectLists; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index dae421e9ff23a6..7215cf0fc7634f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -415,7 +415,8 @@ public class SchemaTable extends Table { .column("ROUTINE_TYPE", ScalarType.createVarchar(64)) .column("DATA_TYPEDTD_IDENDS", ScalarType.createVarchar(64)) .build())) - .put("simple_tables", new SchemaTable(SystemIdGenerator.getNextId(), "simple_tables", TableType.SCHEMA, + .put("metadata_name_ids", new SchemaTable(SystemIdGenerator.getNextId(), + "metadata_name_ids", TableType.SCHEMA, builder().column("CATALOG_ID", ScalarType.createType(PrimitiveType.BIGINT)) .column("CATALOG_NAME", ScalarType.createVarchar(FN_REFLEN)) .column("DATABASE_ID", ScalarType.createType(PrimitiveType.BIGINT)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 1e011d702f3529..86d979188299a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -67,7 +67,6 @@ public class SessionVariable implements Serializable, Writable { public static final String SCAN_QUEUE_MEM_LIMIT = "scan_queue_mem_limit"; public static final String QUERY_TIMEOUT = "query_timeout"; - public static final String QUERY_SIMPLE_TABLES_TIMEOUT = "query_simple_tables_timeout"; public static final String MAX_EXECUTION_TIME = "max_execution_time"; public static final String INSERT_TIMEOUT = "insert_timeout"; public static final String ENABLE_PROFILE = "enable_profile"; @@ -444,10 +443,6 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = INSERT_TIMEOUT) public int insertTimeoutS = 14400; - //query internal.information_schema.simple_tables time - @VariableMgr.VarAttr(name = QUERY_SIMPLE_TABLES_TIMEOUT) - public int querySimpleTablesTimeout = 3; - // if true, need report to coordinator when plan fragment execute successfully. @VariableMgr.VarAttr(name = ENABLE_PROFILE, needForward = true) public boolean enableProfile = false; @@ -729,6 +724,7 @@ public class SessionVariable implements Serializable, Writable { public int getBeNumberForTest() { return beNumberForTest; } + @VariableMgr.VarAttr(name = PROFILLING) public boolean profiling = false; @@ -1316,9 +1312,6 @@ public int getInsertTimeoutS() { return insertTimeoutS; } - public int getQuerySimpleTablesTimeout() { - return querySimpleTablesTimeout; - } public void setInsertTimeoutS(int insertTimeoutS) { this.insertTimeoutS = insertTimeoutS; diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 842e14ef34ed96..0bbcb76d5de707 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -145,7 +145,7 @@ import org.apache.doris.thrift.TInitExternalCtlMetaRequest; import org.apache.doris.thrift.TInitExternalCtlMetaResult; import org.apache.doris.thrift.TListPrivilegesResult; -import org.apache.doris.thrift.TListSimpleTableStatusResult; +import org.apache.doris.thrift.TListTableMetadataNameIdsResult; import org.apache.doris.thrift.TListTableStatusResult; import org.apache.doris.thrift.TLoadTxn2PCRequest; import org.apache.doris.thrift.TLoadTxn2PCResult; @@ -178,7 +178,6 @@ import org.apache.doris.thrift.TRollbackTxnResult; import org.apache.doris.thrift.TShowVariableRequest; import org.apache.doris.thrift.TShowVariableResult; -import org.apache.doris.thrift.TSimpleTableStatus; import org.apache.doris.thrift.TSnapshotLoaderReportRequest; import org.apache.doris.thrift.TSnapshotType; import org.apache.doris.thrift.TStatus; @@ -189,6 +188,7 @@ import org.apache.doris.thrift.TStreamLoadWithLoadStatusRequest; import org.apache.doris.thrift.TStreamLoadWithLoadStatusResult; import org.apache.doris.thrift.TTableIndexQueryStats; +import org.apache.doris.thrift.TTableMetadataNameIds; import org.apache.doris.thrift.TTableQueryStats; import org.apache.doris.thrift.TTableStatus; import org.apache.doris.thrift.TUniqueId; @@ -697,14 +697,14 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE return result; } - public TListSimpleTableStatusResult listSimpleTableStatus(TGetTablesParams params) throws TException { + public TListTableMetadataNameIdsResult listTableMetadataNameIds(TGetTablesParams params) throws TException { - long timeoutTs = System.currentTimeMillis() + exeEnv.getScheduler().getContext(0) - .getSessionVariable().getQuerySimpleTablesTimeout() * 1000L; + + long timeoutTs = System.currentTimeMillis() + Config.query_metadata_name_ids_timeout * 1000L; LOG.debug("get list simple table request: {}", params); - TListSimpleTableStatusResult result = new TListSimpleTableStatusResult(); - List tablesResult = Lists.newArrayList(); + TListTableMetadataNameIdsResult result = new TListTableMetadataNameIdsResult(); + List tablesResult = Lists.newArrayList(); result.setTables(tablesResult); PatternMatcher matcher = null; if (params.isSetPattern()) { @@ -743,7 +743,7 @@ public TListSimpleTableStatusResult listSimpleTableStatus(TGetTablesParams param if (matcher != null && !matcher.match(table.getName())) { continue; } - TSimpleTableStatus status = new TSimpleTableStatus(); + TTableMetadataNameIds status = new TTableMetadataNameIds(); status.setName(table.getName()); status.setId(table.getId()); diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index ec9269ac7a2070..30205708768544 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -118,8 +118,8 @@ enum TSchemaTableType { SCH_BACKENDS, SCH_COLUMN_STATISTICS, SCH_PARAMETERS, - SCH_SIMPLE_TABLES, - SCH_PROFILING; + SCH_METADATA_NAME_IDS, + SCH_PROFILING; } enum THdfsCompression { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 1e16b657d8630b..29fa9871ac458f 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -353,12 +353,13 @@ struct TListTableStatusResult { 1: required list tables } -struct TSimpleTableStatus { - 1: required string name - 2: required i64 id +struct TTableMetadataNameIds { + 1: optional string name + 2: optional i64 id } -struct TListSimpleTableStatusResult { - 1: required list tables + +struct TListTableMetadataNameIdsResult { + 1: optional list tables } // getTableNames returns a list of unqualified table names @@ -1144,7 +1145,7 @@ service FrontendService { TMasterOpResult forward(1: TMasterOpRequest params) TListTableStatusResult listTableStatus(1: TGetTablesParams params) - TListSimpleTableStatusResult listSimpleTableStatus(1: TGetTablesParams params) + TListTableMetadataNameIdsResult listTableMetadataNameIds(1: TGetTablesParams params) TListPrivilegesResult listTablePrivilegeStatus(1: TGetTablesParams params) TListPrivilegesResult listSchemaPrivilegeStatus(1: TGetTablesParams params) TListPrivilegesResult listUserPrivilegeStatus(1: TGetTablesParams params) From 8c37cfd51eab646b60bd5fa83b8d3f65f31fcbb6 Mon Sep 17 00:00:00 2001 From: changyuwei <2017501503@qq.com> Date: Fri, 11 Aug 2023 15:59:04 +0800 Subject: [PATCH 4/8] [feature](information_schema)add information_schema.simple_tables for quickly get catlogs,db,table. --- .../system/test_metadata_name_ids.out | 19 +++++++ .../query_p0/system/test_query_sys_tables.out | 1 + .../system/test_metadata_name_ids.groovy | 50 +++++++++++++++++++ .../system/test_query_sys_tables.groovy | 1 + 4 files changed, 71 insertions(+) create mode 100644 regression-test/data/query_p0/system/test_metadata_name_ids.out create mode 100644 regression-test/suites/query_p0/system/test_metadata_name_ids.groovy diff --git a/regression-test/data/query_p0/system/test_metadata_name_ids.out b/regression-test/data/query_p0/system/test_metadata_name_ids.out new file mode 100644 index 00000000000000..99b799f3649ffa --- /dev/null +++ b/regression-test/data/query_p0/system/test_metadata_name_ids.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc -- +CATALOG_ID BIGINT Yes false \N +CATALOG_NAME VARCHAR(512) Yes false \N +DATABASE_ID BIGINT Yes false \N +DATABASE_NAME VARCHAR(64) Yes false \N +TABLE_ID BIGINT Yes false \N +TABLE_NAME VARCHAR(64) Yes false \N + +-- !select1 -- +internal __internal_schema column_statistics +internal __internal_schema histogram_statistics +internal __internal_schema table_statistics + +-- !select2 -- +internal demo test_metadata_name_ids + +-- !select3 -- + diff --git a/regression-test/data/query_p0/system/test_query_sys_tables.out b/regression-test/data/query_p0/system/test_query_sys_tables.out index 73300030c84d65..7b7ce0583de21d 100644 --- a/regression-test/data/query_p0/system/test_query_sys_tables.out +++ b/regression-test/data/query_p0/system/test_query_sys_tables.out @@ -57,3 +57,4 @@ test_view -- !sql -- +-- !sql -- diff --git a/regression-test/suites/query_p0/system/test_metadata_name_ids.groovy b/regression-test/suites/query_p0/system/test_metadata_name_ids.groovy new file mode 100644 index 00000000000000..333303120afd01 --- /dev/null +++ b/regression-test/suites/query_p0/system/test_metadata_name_ids.groovy @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_metadata_name_ids", "p0" ) { + + + def tableName = "internal.information_schema.metadata_name_ids" + qt_desc """ desc ${tableName} """ + + qt_select1 """ select CATALOG_NAME,DATABASE_NAME,TABLE_NAME from ${tableName} + where CATALOG_NAME="internal" and DATABASE_NAME ="__internal_schema" order by TABLE_NAME ; + """ + + sql """ create database if not exists demo; """ + sql """ use demo ; """ + + sql """ create table if not exists test_metadata_name_ids ( + a int , + b varchar(30) + ) + DUPLICATE KEY(`a`) + DISTRIBUTED BY HASH(`a`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + qt_select2 """ select CATALOG_NAME,DATABASE_NAME,TABLE_NAME from ${tableName} + where CATALOG_NAME="internal" and DATABASE_NAME ="demo" and TABLE_NAME="test_metadata_name_ids";""" + + sql """ drop table test_metadata_name_ids """ + + qt_select3 """ select CATALOG_NAME,DATABASE_NAME,TABLE_NAME from ${tableName} + where CATALOG_NAME="internal" and DATABASE_NAME ="demo" and TABLE_NAME="test_metadata_name_ids";""" + + +} diff --git a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy index f74edcd52406ff..c848f91ebca26f 100644 --- a/regression-test/suites/query_p0/system/test_query_sys_tables.groovy +++ b/regression-test/suites/query_p0/system/test_query_sys_tables.groovy @@ -206,4 +206,5 @@ suite("test_query_sys_tables", "query,p0") { qt_sql "select * from key_column_usage" qt_sql "select * from triggers" qt_sql "select * from parameters" + qt_sql "select * from profiling" } \ No newline at end of file From 82ef53c5fcb9e3d13023d6f5bcc8b5c7ba80bd78 Mon Sep 17 00:00:00 2001 From: changyuwei <2017501503@qq.com> Date: Fri, 18 Aug 2023 15:06:20 +0800 Subject: [PATCH 5/8] [feature](information_schema)add information_schema.simple_tables for quickly get catlogs,db,table. --- .../doris/service/FrontendServiceImpl.java | 69 ++++++++++++------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 0bbcb76d5de707..20549020e46030 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -226,6 +226,10 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.IntSupplier; @@ -699,22 +703,11 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE public TListTableMetadataNameIdsResult listTableMetadataNameIds(TGetTablesParams params) throws TException { - - long timeoutTs = System.currentTimeMillis() + Config.query_metadata_name_ids_timeout * 1000L; - LOG.debug("get list simple table request: {}", params); + TListTableMetadataNameIdsResult result = new TListTableMetadataNameIdsResult(); List tablesResult = Lists.newArrayList(); result.setTables(tablesResult); - PatternMatcher matcher = null; - if (params.isSetPattern()) { - try { - matcher = PatternMatcher.createMysqlPattern(params.getPattern(), - CaseSensibility.TABLE.getCaseSensibility()); - } catch (PatternMatcherException e) { - throw new TException("Pattern is in bad format " + params.getPattern()); - } - } UserIdentity currentUser; if (params.isSetCurrentUserIdent()) { @@ -723,15 +716,32 @@ public TListTableMetadataNameIdsResult listTableMetadataNameIds(TGetTablesParams currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip); } - String catalogName = InternalCatalog.INTERNAL_CATALOG_NAME; + String catalogName; if (params.isSetCatalog()) { catalogName = params.catalog; + } else { + catalogName = InternalCatalog.INTERNAL_CATALOG_NAME; } - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName); - if (catalog != null) { - DatabaseIf db = catalog.getDbNullable(params.db); - if (db != null) { - try { + + PatternMatcher matcher = null; + if (params.isSetPattern()) { + try { + matcher = PatternMatcher.createMysqlPattern(params.getPattern(), + CaseSensibility.TABLE.getCaseSensibility()); + } catch (PatternMatcherException e) { + throw new TException("Pattern is in bad format " + params.getPattern()); + } + } + PatternMatcher finalMatcher = matcher; + + + ExecutorService executor = Executors.newSingleThreadExecutor(); + Future future = executor.submit(() -> { + + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName); + if (catalog != null) { + DatabaseIf db = catalog.getDbNullable(params.db); + if (db != null) { List tables = db.getTables(); for (TableIf table : tables) { if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(currentUser, params.db, @@ -740,7 +750,7 @@ public TListTableMetadataNameIdsResult listTableMetadataNameIds(TGetTablesParams } table.readLock(); try { - if (matcher != null && !matcher.match(table.getName())) { + if (finalMatcher != null && !finalMatcher.match(table.getName())) { continue; } TTableMetadataNameIds status = new TTableMetadataNameIds(); @@ -751,16 +761,23 @@ public TListTableMetadataNameIdsResult listTableMetadataNameIds(TGetTablesParams } finally { table.readUnlock(); } - - long currentTs = System.currentTimeMillis(); - if (currentTs >= timeoutTs) { //limit query time - break; - } } - } catch (Exception e) { - LOG.warn("failed to get tables for db {} in catalog {}", db.getFullName(), catalogName, e); } } + }); + try { + if (catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { + future.get(); + } else { + future.get(Config.query_metadata_name_ids_timeout, TimeUnit.SECONDS); + } + } catch (TimeoutException e) { + future.cancel(true); + LOG.info("From catalog:{},db:{} get tables timeout.", catalogName, params.db); + } catch (InterruptedException | ExecutionException e) { + future.cancel(true); + } finally { + executor.shutdown(); } return result; } From 26ef47aca3c1f81176fb04990fb1897d217afa56 Mon Sep 17 00:00:00 2001 From: changyuwei <2017501503@qq.com> Date: Fri, 18 Aug 2023 19:30:36 +0800 Subject: [PATCH 6/8] [feature](information_schema)add information_schema.simple_tables for quickly get catlogs,db,table. --- be/src/exec/schema_scanner/schema_helper.h | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/exec/schema_scanner/schema_helper.h b/be/src/exec/schema_scanner/schema_helper.h index 590afb4ecf2e17..900f963f7893cc 100644 --- a/be/src/exec/schema_scanner/schema_helper.h +++ b/be/src/exec/schema_scanner/schema_helper.h @@ -34,6 +34,7 @@ class TGetTablesParams; class TGetTablesResult; class TListPrivilegesResult; class TListTableStatusResult; +class TListTableMetadataNameIdsResult; class TShowVariableRequest; class TShowVariableResult; From 86b9d071e02bc827b690e601192383de53c49644 Mon Sep 17 00:00:00 2001 From: changyuwei <2017501503@qq.com> Date: Wed, 23 Aug 2023 14:30:53 +0800 Subject: [PATCH 7/8] [feature](information_schema)add information_schema.simple_tables for quickly get catlogs,db,table. --- .../schema_metadata_name_ids_scanner.cpp | 15 ++++++++++++--- .../doris/service/FrontendServiceImpl.java | 16 ++++++++++++---- gensrc/thrift/FrontendService.thrift | 1 + 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp index 805fc52df69ca2..f99d05dc276bc3 100644 --- a/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp @@ -76,7 +76,7 @@ Status SchemaMetadataNameIdsScanner::start(RuntimeState* state) { db_params.__set_user_ip(*(_param->user_ip)); } } - + db_params.__set_get_null_catalog(true); if (nullptr != _param->ip && 0 != _param->port) { RETURN_IF_ERROR( SchemaHelper::get_db_names(*(_param->ip), _param->port, db_params, &_db_result)); @@ -88,6 +88,14 @@ Status SchemaMetadataNameIdsScanner::start(RuntimeState* state) { Status SchemaMetadataNameIdsScanner::_get_new_table() { SCOPED_TIMER(_get_table_timer); + if (_db_result.db_ids[_db_index] == -1 && + _db_result.dbs[_db_index] == "NULL") { //catalog is empty. + _db_index++; + _table_result.tables.clear(); + _table_result.tables.push_back(TTableMetadataNameIds()); + + return Status::OK(); + } TGetTablesParams table_params; table_params.__set_db(_db_result.dbs[_db_index]); if (_db_result.__isset.catalogs) { @@ -120,8 +128,9 @@ Status SchemaMetadataNameIdsScanner::_get_new_table() { Status SchemaMetadataNameIdsScanner::_fill_block_impl(vectorized::Block* block) { SCOPED_TIMER(_fill_block_timer); auto table_num = _table_result.tables.size(); - if (table_num == 0) { - return Status::OK(); + if (table_num == 0) { //database is null + table_num = 1; + _table_result.tables.push_back(TTableMetadataNameIds()); } std::vector null_datas(table_num, nullptr); std::vector datas(table_num); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 20549020e46030..16b45e5844772b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -221,6 +221,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -363,17 +364,24 @@ public TGetDbsResult getDbNames(TGetDbsParams params) throws TException { .getCatalogOrException(params.catalog, catalog -> new TException("Unknown catalog " + catalog))); } for (CatalogIf catalog : catalogIfs) { - Collection dbs; + Collection dbs = new HashSet(); try { dbs = catalog.getAllDbs(); } catch (Exception e) { LOG.warn("failed to get database names for catalog {}", catalog.getName(), e); // Some external catalog may fail to get databases due to wrong connection info. - // So continue here to get databases of other catalogs. - continue; } LOG.debug("get db size: {}, in catalog: {}", dbs.size(), catalog.getName()); - + if (dbs.isEmpty() && params.isSetGetNullCatalog() && params.get_null_catalog) { + catalogNames.add(catalog.getName()); + dbNames.add("NULL"); + catalogIds.add(catalog.getId()); + dbIds.add(-1L); + continue; + } + if (dbs.isEmpty()) { + continue; + } UserIdentity currentUser = null; if (params.isSetCurrentUserIdent()) { currentUser = UserIdentity.fromThrift(params.current_user_ident); diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 29fa9871ac458f..92adde6cea23eb 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -308,6 +308,7 @@ struct TGetDbsParams { 3: optional string user_ip // deprecated 4: optional Types.TUserIdentity current_user_ident // to replace the user and user ip 5: optional string catalog + 6: optional bool get_null_catalog //if catalog is empty , get dbName ="NULL" and dbId = -1. } // getDbNames returns a list of database names , database ids and catalog names ,catalog ids From b450f3cc97527ec295b1e52b2f08c946b778e6fc Mon Sep 17 00:00:00 2001 From: changyuwei <2017501503@qq.com> Date: Fri, 25 Aug 2023 21:35:32 +0800 Subject: [PATCH 8/8] fix regression test --- .../data/query_p0/system/test_metadata_name_ids.out | 5 ----- .../suites/query_p0/system/test_metadata_name_ids.groovy | 3 --- 2 files changed, 8 deletions(-) diff --git a/regression-test/data/query_p0/system/test_metadata_name_ids.out b/regression-test/data/query_p0/system/test_metadata_name_ids.out index 99b799f3649ffa..4dc532f4d21ec2 100644 --- a/regression-test/data/query_p0/system/test_metadata_name_ids.out +++ b/regression-test/data/query_p0/system/test_metadata_name_ids.out @@ -7,11 +7,6 @@ DATABASE_NAME VARCHAR(64) Yes false \N TABLE_ID BIGINT Yes false \N TABLE_NAME VARCHAR(64) Yes false \N --- !select1 -- -internal __internal_schema column_statistics -internal __internal_schema histogram_statistics -internal __internal_schema table_statistics - -- !select2 -- internal demo test_metadata_name_ids diff --git a/regression-test/suites/query_p0/system/test_metadata_name_ids.groovy b/regression-test/suites/query_p0/system/test_metadata_name_ids.groovy index 333303120afd01..ddce7c6a218a64 100644 --- a/regression-test/suites/query_p0/system/test_metadata_name_ids.groovy +++ b/regression-test/suites/query_p0/system/test_metadata_name_ids.groovy @@ -21,9 +21,6 @@ suite("test_metadata_name_ids", "p0" ) { def tableName = "internal.information_schema.metadata_name_ids" qt_desc """ desc ${tableName} """ - qt_select1 """ select CATALOG_NAME,DATABASE_NAME,TABLE_NAME from ${tableName} - where CATALOG_NAME="internal" and DATABASE_NAME ="__internal_schema" order by TABLE_NAME ; - """ sql """ create database if not exists demo; """ sql """ use demo ; """