From c988b6716ba4cdb6b1848d7bc8871a5d5c8360f4 Mon Sep 17 00:00:00 2001 From: Yulei-Yang Date: Thu, 7 Dec 2023 22:15:27 +0800 Subject: [PATCH 1/2] [Improvement](hms catalog) support show_create_database for hms catalog --- .../Show-Statements/SHOW-CREATE-DATABASE.md | 19 ++++++-- .../Show-Statements/SHOW-CREATE-DATABASE.md | 17 +++++-- .../hive/PooledHiveMetaStoreClient.java | 14 ++++++ .../org/apache/doris/qe/ShowExecutor.java | 27 ++++++++--- .../hive/metastore/HiveMetaStoreClient.java | 7 ++- .../test_show_create_database.groovy | 46 +++++++++++++++++++ 6 files changed, 116 insertions(+), 14 deletions(-) create mode 100644 regression-test/suites/export_p0/test_show_create_database.groovy diff --git a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-CREATE-DATABASE.md b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-CREATE-DATABASE.md index 78f6515e34d9aa..24eba90b290418 100644 --- a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-CREATE-DATABASE.md +++ b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-CREATE-DATABASE.md @@ -32,7 +32,7 @@ SHOW CREATE DATABASE ### Description -This statement checks the creation of the doris database. +This statement checks the creation of the doris database, support database from both internal catalog and hms catalog grammar: @@ -42,11 +42,12 @@ SHOW CREATE DATABASE db_name; illustrate: -- `db_name`: The name of the database where doris exists. +- `db_name`: The name of the database +- if specific a database from hms catalog, will return same with this stmt in hive ### Example -1. View the creation of the test database in doris +1. View the creation of the test database in doris internal catalog ```sql mysql> SHOW CREATE DATABASE test; @@ -58,6 +59,18 @@ illustrate: 1 row in set (0.00 sec) ```` +2. view a database named `hdfs_text` from a hms catalog + + ```sql + mysql> show create database hdfs_text; + +-----------+------------------------------------------------------------------------------------+ + | Database | Create Database | + +-----------+------------------------------------------------------------------------------------+ + | hdfs_text | CREATE DATABASE `hdfs_text` LOCATION 'hdfs://HDFS1009138/hive/warehouse/hdfs_text' | + +-----------+------------------------------------------------------------------------------------+ + 1 row in set (0.01 sec) + ``` + ### Keywords SHOW, CREATE, DATABASE diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-CREATE-DATABASE.md b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-CREATE-DATABASE.md index e46b94a2bd1e3e..331d95270e063f 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-CREATE-DATABASE.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-CREATE-DATABASE.md @@ -32,7 +32,7 @@ SHOW CREATE DATABASE ### Description -该语句查看doris数据库的创建情况。 +该语句查看 doris 内置数据库和 hms catalog 数据库的创建信息。 语法： @@ -42,7 +42,8 @@ SHOW CREATE DATABASE db_name; 说明： -- `db_name`: 为doris存在的数据库名称。 +- `db_name`: 为 内置数据库或 hms catalog 数据库的名称。 +- 如果查看 hms catalog 内数据库，返回信息和 hive 中同名命令结果一样。 ### Example @@ -57,7 +58,17 @@ SHOW CREATE DATABASE db_name; +----------+------------------------+ 1 row in set (0.00 sec) ``` - +2. 查看 hive catalog 中数据库hdfs_text的创建信息 + + ```sql + mysql> show create database hdfs_text; + +-----------+------------------------------------------------------------------------------------+ + | Database | Create Database | + +-----------+------------------------------------------------------------------------------------+ + | hdfs_text | CREATE DATABASE `hdfs_text` LOCATION 'hdfs://HDFS1009138/hive/warehouse/hdfs_text' | + +-----------+------------------------------------------------------------------------------------+ + 1 row in set (0.01 sec) + ``` ### Keywords SHOW, CREATE, DATABASE diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java index c699be330a101a..4328ddc4da6b88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.DataOperationType; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockResponse; @@ -185,6 +186,19 @@ public List getPartitionsByFilter(String dbName, String tblName, Stri } } + public Database getDatabase(String dbName) { + try (CachedClient client = getClient()) { + try { + return client.client.getDatabase(dbName); + } catch (Exception e) { + client.setThrowable(e); + throw e; + } + } catch (Exception e) { + throw new HMSClientException("failed to get database %s from hms client", e, dbName); + } + } + public Table getTable(String dbName, String tblName) { try (CachedClient client = getClient()) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 430f790ecd3768..eee1257733cb78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -935,15 +935,30 @@ private void handleShowVariables() throws AnalysisException { private void handleShowCreateDb() throws AnalysisException { ShowCreateDbStmt showStmt = (ShowCreateDbStmt) stmt; List> rows = Lists.newArrayList(); - DatabaseIf db = ctx.getCurrentCatalog().getDbOrAnalysisException(showStmt.getDb()); + StringBuilder sb = new StringBuilder(); - sb.append("CREATE DATABASE `").append(ClusterNamespace.getNameFromFullName(showStmt.getDb())).append("`"); - if (db.getDbProperties().getProperties().size() > 0) { - sb.append("\nPROPERTIES (\n"); - sb.append(new PrintableMap<>(db.getDbProperties().getProperties(), "=", true, true, false)); - sb.append("\n)"); + CatalogIf catalog = ctx.getCurrentCatalog(); + if (catalog.isInternalCatalog()) { + DatabaseIf db = catalog.getDbOrAnalysisException(showStmt.getDb()); + sb.append("CREATE DATABASE `").append(ClusterNamespace.getNameFromFullName(showStmt.getDb())).append("`"); + if (db.getDbProperties().getProperties().size() > 0) { + sb.append("\nPROPERTIES (\n"); + sb.append(new PrintableMap<>(db.getDbProperties().getProperties(), "=", true, true, false)); + sb.append("\n)"); + } + } else if (catalog instanceof HMSExternalCatalog) { + String simpleDBName = ClusterNamespace.getNameFromFullName(showStmt.getDb()); + org.apache.hadoop.hive.metastore.api.Database db = ((HMSExternalCatalog) catalog).getClient() + .getDatabase(simpleDBName); + sb.append("CREATE DATABASE `").append(simpleDBName).append("`") + .append(" LOCATION '") + .append(db.getLocationUri()) + .append("'"); + } else { + throw new AnalysisException("Only support internal catalog and hms catalog"); } + rows.add(Lists.newArrayList(ClusterNamespace.getNameFromFullName(showStmt.getDb()), sb.toString())); resultSet = new ShowResultSet(showStmt.getMetaData(), rows); } diff --git a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index ff85e31959769a..3fd0379194f39d 100644 --- a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -1722,8 +1722,11 @@ public Database getDatabase(String name) throws TException { @Override public Database getDatabase(String catalogName, String databaseName) throws TException { - Database d = client.get_database(prependCatalogToDbName(catalogName, databaseName, conf)); - return deepCopy(filterHook.filterDatabase(d)); + if (hiveVersion == HiveVersion.V1_0 || hiveVersion == HiveVersion.V2_0 || hiveVersion == HiveVersion.V2_3) { + return deepCopy(client.get_database(databaseName)); + } else { + return deepCopy(client.get_database(prependCatalogToDbName(catalogName, databaseName, conf))); + } } @Override diff --git a/regression-test/suites/export_p0/test_show_create_database.groovy b/regression-test/suites/export_p0/test_show_create_database.groovy new file mode 100644 index 00000000000000..41c19398cfaaaf --- /dev/null +++ b/regression-test/suites/export_p0/test_show_create_database.groovy @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_show_create_database", "p0,external,hive,external_docker,external_docker_hive") { + + sql """create database if not exists db_test""" + result = sql """show create database db_test""" + assertEquals(result.size(), 1) + assertEquals(result[0][1], "CREATE DATABASE `db_test`") + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String hms_port = context.config.otherConfigs.get("hms_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + String catalog_name = "hive_test_other" + + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + + sql """switch ${catalog_name}""" + + result = sql """show create database `default`""" + assertEquals(result.size(), 1) + assertTrue(result[0][1].contains("CREATE DATABASE `default` LOCATION 'hdfs:")) + + sql """drop catalog if exists ${catalog_name}""" + } +} \ No newline at end of file From 19e813a0d8a52196017583eee1a65d85a6a38a5b Mon Sep 17 00:00:00 2001 From: Yulei-Yang Date: Fri, 8 Dec 2023 10:54:02 +0800 Subject: [PATCH 2/2] update --- .../java/org/apache/doris/qe/ShowExecutor.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index eee1257733cb78..0ce3317ec14d22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -938,15 +938,7 @@ private void handleShowCreateDb() throws AnalysisException { StringBuilder sb = new StringBuilder(); CatalogIf catalog = ctx.getCurrentCatalog(); - if (catalog.isInternalCatalog()) { - DatabaseIf db = catalog.getDbOrAnalysisException(showStmt.getDb()); - sb.append("CREATE DATABASE `").append(ClusterNamespace.getNameFromFullName(showStmt.getDb())).append("`"); - if (db.getDbProperties().getProperties().size() > 0) { - sb.append("\nPROPERTIES (\n"); - sb.append(new PrintableMap<>(db.getDbProperties().getProperties(), "=", true, true, false)); - sb.append("\n)"); - } - } else if (catalog instanceof HMSExternalCatalog) { + if (catalog instanceof HMSExternalCatalog) { String simpleDBName = ClusterNamespace.getNameFromFullName(showStmt.getDb()); org.apache.hadoop.hive.metastore.api.Database db = ((HMSExternalCatalog) catalog).getClient() .getDatabase(simpleDBName); @@ -955,7 +947,13 @@ private void handleShowCreateDb() throws AnalysisException { .append(db.getLocationUri()) .append("'"); } else { - throw new AnalysisException("Only support internal catalog and hms catalog"); + DatabaseIf db = catalog.getDbOrAnalysisException(showStmt.getDb()); + sb.append("CREATE DATABASE `").append(ClusterNamespace.getNameFromFullName(showStmt.getDb())).append("`"); + if (db.getDbProperties().getProperties().size() > 0) { + sb.append("\nPROPERTIES (\n"); + sb.append(new PrintableMap<>(db.getDbProperties().getProperties(), "=", true, true, false)); + sb.append("\n)"); + } }