From ef2193be32d303c670f571cf2403279630d90e15 Mon Sep 17 00:00:00 2001 From: Yulei-Yang Date: Fri, 14 Jun 2024 11:06:29 +0800 Subject: [PATCH] [Improvement](hms catalog) support show_create_database for hms catalog --- .../hive/PooledHiveMetaStoreClient.java | 14 ++++++ .../org/apache/doris/qe/ShowExecutor.java | 25 +++++++--- .../hive/metastore/HiveMetaStoreClient.java | 7 ++- .../test_show_create_database.groovy | 46 +++++++++++++++++++ 4 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 regression-test/suites/export_p0/test_show_create_database.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java index 8023a74e31a8df..b8c515f45d3247 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/PooledHiveMetaStoreClient.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.DataOperationType; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockResponse; @@ -188,6 +189,19 @@ public List getPartitionsByFilter(String dbName, String tblName, Stri } } + public Database getDatabase(String dbName) { + try (CachedClient client = getClient()) { + try { + return client.client.getDatabase(dbName); + } catch (Exception e) { + client.setThrowable(e); + throw e; + } + } catch (Exception e) { + throw new HMSClientException("failed to get database %s from hms client", e, dbName); + } + } + public Table getTable(String dbName, String tblName) { try (CachedClient client = getClient()) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index e7350a1c30ae23..4726d036619733 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -991,15 +991,28 @@ private void handleShowVariables() throws AnalysisException { private void handleShowCreateDb() throws AnalysisException { ShowCreateDbStmt showStmt = (ShowCreateDbStmt) stmt; List> rows = Lists.newArrayList(); - DatabaseIf db = ctx.getCurrentCatalog().getDbOrAnalysisException(showStmt.getDb()); + StringBuilder sb = new StringBuilder(); - sb.append("CREATE DATABASE `").append(ClusterNamespace.getNameFromFullName(showStmt.getDb())).append("`"); - if (db.getDbProperties().getProperties().size() > 0) { - sb.append("\nPROPERTIES (\n"); - sb.append(new PrintableMap<>(db.getDbProperties().getProperties(), "=", true, true, false)); - sb.append("\n)"); + CatalogIf catalog = ctx.getCurrentCatalog(); + if (catalog instanceof HMSExternalCatalog) { + String simpleDBName = ClusterNamespace.getNameFromFullName(showStmt.getDb()); + org.apache.hadoop.hive.metastore.api.Database db = ((HMSExternalCatalog) catalog).getClient() + .getDatabase(simpleDBName); + sb.append("CREATE DATABASE `").append(simpleDBName).append("`") + .append(" LOCATION '") + .append(db.getLocationUri()) + .append("'"); + } else { + DatabaseIf db = catalog.getDbOrAnalysisException(showStmt.getDb()); + sb.append("CREATE DATABASE `").append(ClusterNamespace.getNameFromFullName(showStmt.getDb())).append("`"); + if (db.getDbProperties().getProperties().size() > 0) { + sb.append("\nPROPERTIES (\n"); + sb.append(new PrintableMap<>(db.getDbProperties().getProperties(), "=", true, true, false)); + sb.append("\n)"); + } } + rows.add(Lists.newArrayList(ClusterNamespace.getNameFromFullName(showStmt.getDb()), sb.toString())); resultSet = new ShowResultSet(showStmt.getMetaData(), rows); } diff --git a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 8adad6d8bf307a..2fb571d8c18107 100644 --- a/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/fe/fe-core/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -1721,8 +1721,11 @@ public Database getDatabase(String name) throws TException { @Override public Database getDatabase(String catalogName, String databaseName) throws TException { - Database d = client.get_database(prependCatalogToDbName(catalogName, databaseName, conf)); - return deepCopy(filterHook.filterDatabase(d)); + if (hiveVersion == HiveVersion.V1_0 || hiveVersion == HiveVersion.V2_0 || hiveVersion == HiveVersion.V2_3) { + return deepCopy(client.get_database(databaseName)); + } else { + return deepCopy(client.get_database(prependCatalogToDbName(catalogName, databaseName, conf))); + } } @Override diff --git a/regression-test/suites/export_p0/test_show_create_database.groovy b/regression-test/suites/export_p0/test_show_create_database.groovy new file mode 100644 index 00000000000000..41c19398cfaaaf --- /dev/null +++ b/regression-test/suites/export_p0/test_show_create_database.groovy @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_show_create_database", "p0,external,hive,external_docker,external_docker_hive") { + + sql """create database if not exists db_test""" + result = sql """show create database db_test""" + assertEquals(result.size(), 1) + assertEquals(result[0][1], "CREATE DATABASE `db_test`") + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String hms_port = context.config.otherConfigs.get("hms_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + String catalog_name = "hive_test_other" + + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + + sql """switch ${catalog_name}""" + + result = sql """show create database `default`""" + assertEquals(result.size(), 1) + assertTrue(result[0][1].contains("CREATE DATABASE `default` LOCATION 'hdfs:")) + + sql """drop catalog if exists ${catalog_name}""" + } +} \ No newline at end of file