diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index 1171adc80a7e4d..f8a3188328391b 100644 --- a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -94,6 +94,7 @@ Syntax: * BITMAP_UNION: Only for BITMAP type Allow NULL: Default is NOT NULL. NULL value should be represented as `\N` in load source file. Notice: + The origin value of BITMAP_UNION column should be TINYINT, SMALLINT, INT, BIGINT. 2. index_definition Syntax: @@ -133,14 +134,14 @@ Syntax: "line_delimiter" = "value_delimiter" ) ``` - + ``` BROKER PROPERTIES( "username" = "name", "password" = "password" ) ``` - + For different broker, the broker properties are different Notice: Files name in "path" is separated by ",". If file name includes ",", use "%2c" instead. If file name includes "%", use "%25" instead. @@ -220,7 +221,7 @@ Syntax: ["replication_num" = "3"] ) ``` - + storage_medium: SSD or HDD, The default initial storage media can be specified by `default_storage_medium= XXX` in the fe configuration file `fe.conf`, or, if not, by default, HDD. Note: when FE configuration 'enable_strict_storage_medium_check' is' True ', if the corresponding storage medium is not set in the cluster, the construction clause 'Failed to find enough host in all backends with storage medium is SSD|HDD'. storage_cooldown_time: If storage_medium is SSD, data will be automatically moved to HDD when timeout. @@ -246,9 +247,9 @@ Syntax: "colocate_with"="table1" ) ``` - + 4) if you want to use the dynamic partitioning feature, specify it in properties - + ``` PROPERTIES ( "dynamic_partition.enable" = "true|false", @@ -268,6 +269,7 @@ Syntax: Dynamic_partition. Prefix: used to specify the partition name prefix to be created, such as the partition name prefix p, automatically creates the partition name p20200108 Dynamic_partition. Buckets: specifies the number of partition buckets that are automatically created + ``` 8. rollup_index grammar: ``` @@ -320,6 +322,7 @@ Syntax: "storage_medium" = "SSD", "storage_cooldown_time" = "2015-06-04 00:00:00" ); + ``` 3. Create an olap table, with range partitioned, distributed by hash. @@ -347,16 +350,16 @@ Syntax: "storage_medium" = "SSD", "storage_cooldown_time" = "2015-06-04 00:00:00" ); ``` - + Explain: This statement will create 3 partitions: - + ``` ( { MIN }, {"2014-01-01"} ) [ {"2014-01-01"}, {"2014-06-01"} ) [ {"2014-06-01"}, {"2014-12-01"} ) ``` - + Data outside these ranges will not be loaded. 2) Fixed Range @@ -381,8 +384,8 @@ Syntax: ); 4. Create a mysql table - - ``` + 4.1 Create MySQL table directly from external table information +``` CREATE EXTERNAL TABLE example_db.table_mysql ( k1 DATE, @@ -400,8 +403,38 @@ Syntax: "password" = "mysql_passwd", "database" = "mysql_db_test", "table" = "mysql_table_test" - ); - ``` + ) +``` + + 4.2 Create MySQL table with external ODBC catalog resource +``` + CREATE EXTERNAL RESOURCE "mysql_resource" + PROPERTIES + ( + "type" = "odbc_catalog", + "user" = "mysql_user", + "password" = "mysql_passwd", + "host" = "127.0.0.1", + "port" = "8239" + ); +``` +``` + CREATE EXTERNAL TABLE example_db.table_mysql + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + k4 VARCHAR(2048), + k5 DATETIME + ) + ENGINE=mysql + PROPERTIES + ( + "odbc_catalog_resource" = "mysql_resource", + "database" = "mysql_db_test", + "table" = "mysql_table_test" + ) +``` 5. Create a broker table, with file on HDFS, line delimit by "|", column separated by "\n" @@ -549,7 +582,7 @@ Syntax: "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32" ); - ``` + ``` 12. Create a table with rollup index ``` CREATE TABLE example_db.rolup_index_table diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index ca223b0de4f731..f1e7c8fa32fb24 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -151,7 +151,7 @@ under the License. 注意： "path" 中如果有多个文件，用逗号[,]分割。如果文件名中包含逗号，那么使用 %2c 来替代。如果文件名中包含 %，使用 %25 代替 现在文件内容格式支持CSV，支持GZ，BZ2，LZ4，LZO(LZOP) 压缩格式。 - + 3) 如果是 hive，则需要在 properties 提供以下信息： ``` PROPERTIES ( @@ -159,7 +159,7 @@ under the License. "table" = "hive_table_name", "hive.metastore.uris" = "thrift://127.0.0.1:9083" ) - + ``` 其中 database 是 hive 表对应的库名字，table 是 hive 表的名字，hive.metastore.uris 是 hive metastore 服务地址。 注意：目前hive外部表仅用于Spark Load使用，不支持查询。 @@ -193,7 +193,7 @@ under the License. ... ) ``` - + 说明： 使用指定的 key 列和指定的数值范围进行分区。 1) 分区名称仅支持字母开头，字母、数字和下划线组成 @@ -202,7 +202,7 @@ under the License. 3) 分区为左闭右开区间，首个分区的左边界为做最小值 4) NULL 值只会存放在包含最小值的分区中。当包含最小值的分区被删除后，NULL 值将无法导入。 5) 可以指定一列或多列作为分区列。如果分区值缺省，则会默认填充最小值。 - + 注意： 1) 分区一般用于时间维度的数据管理 2) 有数据回溯需求的，可以考虑首个分区为空分区，以便后续增加分区 @@ -270,9 +270,9 @@ under the License. "colocate_with"="table1" ) ``` - + 4) 如果希望使用动态分区特性，需要在properties 中指定 - + ``` PROPERTIES ( "dynamic_partition.enable" = "true|false", @@ -288,7 +288,7 @@ under the License. dynamic_partition.end: 用于指定提前创建的分区数量。值必须大于0。 dynamic_partition.prefix: 用于指定创建的分区名前缀，例如分区名前缀为p，则自动创建分区名为p20200108 dynamic_partition.buckets: 用于指定自动创建的分区分桶数量 - + 5) 建表时可以批量创建多个 Rollup 语法： ``` @@ -296,7 +296,7 @@ under the License. [FROM from_index_name] [PROPERTIES ("key"="value", ...)],...) ``` - + 6) 如果希望使用 内存表 特性，需要在 properties 中指定 ``` @@ -419,6 +419,7 @@ under the License. 4. 创建一个 mysql 表 + 4.1 直接通过外表信息创建mysql表 ``` CREATE EXTERNAL TABLE example_db.table_mysql ( @@ -440,6 +441,36 @@ under the License. ) ``` + 4.2 通过External Catalog Resource创建mysql表 +``` + CREATE EXTERNAL RESOURCE "mysql_resource" + PROPERTIES + ( + "type" = "odbc_catalog", + "user" = "mysql_user", + "password" = "mysql_passwd", + "host" = "127.0.0.1", + "port" = "8239" + ); +``` +``` + CREATE EXTERNAL TABLE example_db.table_mysql + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + k4 VARCHAR(2048), + k5 DATETIME + ) + ENGINE=mysql + PROPERTIES + ( + "odbc_catalog_resource" = "mysql_resource", + "database" = "mysql_db_test", + "table" = "mysql_table_test" + ) +``` + 5. 创建一个数据文件存储在HDFS上的 broker 外部表, 数据使用 "|" 分割，"\n" 换行 ``` @@ -650,3 +681,5 @@ under the License. ## keyword CREATE,TABLE + +``` diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java index 1a1c88df3a1aa9..edb39ffcbd22c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java @@ -215,7 +215,7 @@ private void analyzeSubPredicate(Expr subExpr) throws AnalysisException { if (!valid) { throw new AnalysisException("Where clause should looks like: NAME = \"your_resource_name\"," - + " or NAME LIKE \"matcher\", " + " or RESOURCETYPE = \"SPARK\", " + + " or NAME LIKE \"matcher\", " + " or RESOURCETYPE = \"resource_type\", " + " or compound predicate with operator AND"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 58a864edd0cadf..8b47b1161171d3 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -4059,10 +4059,14 @@ public static void getDdlStmt(Table table, List createTableStmt, List createTableStmt, List schema, Map private void validate(Map properties) throws DdlException { if (properties == null) { throw new DdlException("Please set properties of mysql table, " - + "they are: host, port, user, password, database and table"); + + "they are: odbc_catalog_resource or [host, port, user, password] and database and table"); } - // Set up - host = properties.get(MYSQL_HOST); - if (Strings.isNullOrEmpty(host)) { - throw new DdlException("Host of MySQL table is null. " - + "Please add properties('host'='xxx.xxx.xxx.xxx') when create table"); - } + if (properties.containsKey(ODBC_CATALOG_RESOURCE)) { + odbcCatalogResourceName = properties.get(ODBC_CATALOG_RESOURCE); + + // 1. check whether resource exist + Resource oriResource = Catalog.getCurrentCatalog().getResourceMgr().getResource(odbcCatalogResourceName); + if (oriResource == null) { + throw new DdlException("Resource does not exist. name: " + odbcCatalogResourceName); + } - port = properties.get(MYSQL_PORT); - if (Strings.isNullOrEmpty(port)) { - // Maybe null pointer or number convert - throw new DdlException("Port of MySQL table is null. " - + "Please add properties('port'='3306') when create table"); + // 2. check resource usage privilege + if (!Catalog.getCurrentCatalog().getAuth().checkResourcePriv(ConnectContext.get(), + odbcCatalogResourceName, + PrivPredicate.USAGE)) { + throw new DdlException("USAGE denied to user '" + ConnectContext.get().getQualifiedUser() + + "'@'" + ConnectContext.get().getRemoteIP() + + "' for resource '" + odbcCatalogResourceName + "'"); + } } else { - try { - Integer.valueOf(port); - } catch (Exception e) { - throw new DdlException("Port of MySQL table must be a number." - + "Please add properties('port'='3306') when create table"); + // Set up + host = properties.get(MYSQL_HOST); + if (Strings.isNullOrEmpty(host)) { + throw new DdlException("Host of MySQL table is null. " + + "Please set proper resource or add properties('host'='xxx.xxx.xxx.xxx') when create table"); + } + port = properties.get(MYSQL_PORT); + if (Strings.isNullOrEmpty(port)) { + // Maybe null pointer or number convert + throw new DdlException("Port of MySQL table is null. " + + "Please set proper resource or add properties('port'='3306') when create table"); + } else { + try { + Integer.valueOf(port); + } catch (Exception e) { + throw new DdlException("Port of MySQL table must be a number." + + "Please set proper resource or add properties('port'='3306') when create table"); + + } } - } - userName = properties.get(MYSQL_USER); - if (Strings.isNullOrEmpty(userName)) { - throw new DdlException("User of MySQL table is null. " - + "Please add properties('user'='root') when create table"); - } + userName = properties.get(MYSQL_USER); + if (Strings.isNullOrEmpty(userName)) { + throw new DdlException("User of MySQL table is null. " + + "Please set proper resource or add properties('user'='root') when create table"); + } - passwd = properties.get(MYSQL_PASSWORD); - if (passwd == null) { - throw new DdlException("Password of MySQL table is null. " - + "Please add properties('password'='xxxx') when create table"); + passwd = properties.get(MYSQL_PASSWORD); + if (passwd == null) { + throw new DdlException("Password of MySQL table is null. " + + "Please set proper resource or add properties('password'='xxxx') when create table"); + } } - + mysqlDatabaseName = properties.get(MYSQL_DATABASE); if (Strings.isNullOrEmpty(mysqlDatabaseName)) { throw new DdlException("Database of MySQL table is null. " @@ -115,21 +140,51 @@ private void validate(Map properties) throws DdlException { + "Please add properties('table'='xxxx') when create table"); } } + + private String getPropertyFromResource(String propertyName) { + OdbcCatalogResource odbcCatalogResource = (OdbcCatalogResource) + (Catalog.getCurrentCatalog().getResourceMgr().getResource(odbcCatalogResourceName)); + if (odbcCatalogResource == null) { + throw new RuntimeException("Resource does not exist. name: " + odbcCatalogResourceName); + } + + String property = odbcCatalogResource.getProperties(propertyName); + if (property == null) { + throw new RuntimeException("The property:" + propertyName + " do not set in resource " + odbcCatalogResourceName); + } + return property; + } + + public String getOdbcCatalogResourceName() { + return odbcCatalogResourceName; + } public String getHost() { - return host; + if (host != null) { + return host; + } + return getPropertyFromResource(MYSQL_HOST); } public String getPort() { - return port; + if (port != null) { + return port; + } + return getPropertyFromResource(MYSQL_PORT); } public String getUserName() { - return userName; + if (userName != null) { + return userName; + } + return getPropertyFromResource(MYSQL_USER); } public String getPasswd() { - return passwd; + if (passwd != null) { + return passwd; + } + return getPropertyFromResource(MYSQL_PASSWORD); } public String getMysqlDatabaseName() { @@ -142,7 +197,7 @@ public String getMysqlTableName() { public TTableDescriptor toThrift() { TMySQLTable tMySQLTable = - new TMySQLTable(host, port, userName, passwd, mysqlDatabaseName, mysqlTableName); + new TMySQLTable(getHost(), getPort(), getUserName(), getPasswd(), mysqlDatabaseName, mysqlTableName); TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.MYSQL_TABLE, fullSchema.size(), 0, getName(), ""); tTableDescriptor.setMysqlTable(tMySQLTable); @@ -161,13 +216,13 @@ public int getSignature(int signatureVersion) { // type adler32.update(type.name().getBytes(charsetName)); // host - adler32.update(host.getBytes(charsetName)); + adler32.update(getHost().getBytes(charsetName)); // port - adler32.update(port.getBytes(charsetName)); + adler32.update(getPort().getBytes(charsetName)); // username - adler32.update(userName.getBytes(charsetName)); + adler32.update(getUserName().getBytes(charsetName)); // passwd - adler32.update(passwd.getBytes(charsetName)); + adler32.update(getPasswd().getBytes(charsetName)); // mysql db adler32.update(mysqlDatabaseName.getBytes(charsetName)); // mysql table @@ -185,23 +240,54 @@ public int getSignature(int signatureVersion) { public void write(DataOutput out) throws IOException { super.write(out); - Text.writeString(out, host); - Text.writeString(out, port); - Text.writeString(out, userName); - Text.writeString(out, passwd); - Text.writeString(out, mysqlDatabaseName); - Text.writeString(out, mysqlTableName); + Map serializeMap = Maps.newHashMap(); + serializeMap.put(ODBC_CATALOG_RESOURCE, odbcCatalogResourceName); + serializeMap.put(MYSQL_HOST, host); + serializeMap.put(MYSQL_PORT, port); + serializeMap.put(MYSQL_USER, userName); + serializeMap.put(MYSQL_PASSWORD, passwd); + serializeMap.put(MYSQL_DATABASE, mysqlDatabaseName); + serializeMap.put(MYSQL_TABLE, mysqlTableName); + + int size = (int) serializeMap.values().stream().filter(v -> { + return v != null; + }).count(); + out.writeInt(size); + for (Map.Entry kv : serializeMap.entrySet()) { + if (kv.getValue() != null) { + Text.writeString(out, kv.getKey()); + Text.writeString(out, kv.getValue()); + } + } } public void readFields(DataInput in) throws IOException { super.readFields(in); - // Read MySQL meta - host = Text.readString(in); - port = Text.readString(in); - userName = Text.readString(in); - passwd = Text.readString(in); - mysqlDatabaseName = Text.readString(in); - mysqlTableName = Text.readString(in); + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_92) { + // Read MySQL meta + int size = in.readInt(); + Map serializeMap = Maps.newHashMap(); + for (int i = 0; i < size; i++) { + String key = Text.readString(in); + String value = Text.readString(in); + serializeMap.put(key, value); + } + + odbcCatalogResourceName = serializeMap.get(ODBC_CATALOG_RESOURCE); + host = serializeMap.get(MYSQL_HOST); + port = serializeMap.get(MYSQL_PORT); + userName = serializeMap.get(MYSQL_USER); + passwd = serializeMap.get(MYSQL_PASSWORD); + mysqlDatabaseName = serializeMap.get(MYSQL_DATABASE); + mysqlTableName = serializeMap.get(MYSQL_TABLE); + } else { + host = Text.readString(in); + port = Text.readString(in); + userName = Text.readString(in); + passwd = Text.readString(in); + mysqlDatabaseName = Text.readString(in); + mysqlTableName = Text.readString(in); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java new file mode 100644 index 00000000000000..4bf547d5e1a8de --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.common.DdlException; +import org.apache.doris.common.proc.BaseProcResult; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; + +import java.util.Map; + +/** + * External ODBC Catalog resource for external table query. + * + * External ODBC Catalog resource example: + * CREATE EXTERNAL RESOURCE "odbc_mysql" + * PROPERTIES + * ( + * "type" = "external_odbc", [required] + * "user" = "root", [required] + * "password" = "root", [required] + * "host" = "192.168.1.1", [required] + * "port" = "8086", [required] + * "odbc_type" = "mysql", [optional, external table of ODBC should set] + * "driver" = "MySQL driver" [optional, external table of ODBC should set] + * ); + * + * DROP RESOURCE "odbc_mysql"; + */ +public class OdbcCatalogResource extends Resource { + // required + private static final String HOST = "host"; + private static final String PORT = "port"; + private static final String USER = "user"; + private static final String PASSWORD = "password"; + + // optional + private static final String TYPE = "odbc_type"; + private static final String DRIVER = "driver"; + + @SerializedName(value = "configs") + private Map configs; + + public OdbcCatalogResource(String name) { + this(name, Maps.newHashMap()); + } + + private OdbcCatalogResource(String name, Map configs) { + super(name, ResourceType.ODBC_CATALOG); + this.configs = configs; + } + + public OdbcCatalogResource getCopiedResource() { + return new OdbcCatalogResource(name, Maps.newHashMap(configs)); + } + + private void checkProperties(String propertieKey) throws DdlException { + // check the propertie key + String value = configs.get(propertieKey); + if (value == null) { + throw new DdlException("Missing " + propertieKey + " in properties"); + } + + } + + public String getProperties(String propertieKey) { + // check the propertie key + String value = configs.get(propertieKey); + return value; + } + + @Override + protected void setProperties(Map properties) throws DdlException { + Preconditions.checkState(properties != null); + + configs = properties; + + checkProperties(HOST); + checkProperties(PORT); + checkProperties(USER); + checkProperties(PASSWORD); + } + + @Override + protected void getProcNodeData(BaseProcResult result) { + String lowerCaseType = type.name().toLowerCase(); + for (Map.Entry entry : configs.entrySet()) { + result.addRow(Lists.newArrayList(name, lowerCaseType, entry.getKey(), entry.getValue())); + } + } +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java index dfd9a6749c7f3d..ee382791511a44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java @@ -20,6 +20,8 @@ import com.google.common.collect.Maps; import org.apache.doris.common.DdlException; import org.apache.doris.common.io.Text; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TOdbcTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; @@ -43,6 +45,7 @@ public class OdbcTable extends Table { private static final Logger LOG = LogManager.getLogger(OlapTable.class); + private static final String ODBC_CATALOG_RESOURCE = "odbc_catalog_resource"; private static final String ODBC_HOST = "host"; private static final String ODBC_PORT = "port"; private static final String ODBC_USER = "user"; @@ -50,7 +53,7 @@ public class OdbcTable extends Table { private static final String ODBC_DATABASE = "database"; private static final String ODBC_TABLE = "table"; private static final String ODBC_DRIVER = "driver"; - private static final String ODBC_TYPE = "type"; + private static final String ODBC_TYPE = "odbc_type"; private static Map TABLE_TYPE_MAP; static { @@ -62,6 +65,7 @@ public class OdbcTable extends Table { TABLE_TYPE_MAP = Collections.unmodifiableMap(tempMap); } + private String odbcCatalogResourceName; private String host; private String port; private String userName; @@ -84,41 +88,79 @@ public OdbcTable(long id, String name, List schema, Map private void validate(Map properties) throws DdlException { if (properties == null) { throw new DdlException("Please set properties of odbc table, " - + "they are: host, port, user, password, database and table"); + + "they are: odbc_catalog_resource or [host, port, user, password, driver, odbc_type]" + + " and database and table"); } - // Set up - host = properties.get(ODBC_HOST); - if (Strings.isNullOrEmpty(host)) { - throw new DdlException("Host of Odbc table is null. " - + "Please add properties('host'='xxx.xxx.xxx.xxx') when create table"); - } + if (properties.containsKey(ODBC_CATALOG_RESOURCE)) { + odbcCatalogResourceName = properties.get(ODBC_CATALOG_RESOURCE); + + // 1. check whether resource exist + Resource oriResource = Catalog.getCurrentCatalog().getResourceMgr().getResource(odbcCatalogResourceName); + if (oriResource == null) { + throw new DdlException("Resource does not exist. name: " + odbcCatalogResourceName); + } - port = properties.get(ODBC_PORT); - if (Strings.isNullOrEmpty(port)) { - // Maybe null pointer or number convert - throw new DdlException("Port of Odbc table is null. " - + "Please add properties('port'='3306') when create table"); + // 2. check resource usage privilege + if (!Catalog.getCurrentCatalog().getAuth().checkResourcePriv(ConnectContext.get(), + odbcCatalogResourceName, + PrivPredicate.USAGE)) { + throw new DdlException("USAGE denied to user '" + ConnectContext.get().getQualifiedUser() + + "'@'" + ConnectContext.get().getRemoteIP() + + "' for resource '" + odbcCatalogResourceName + "'"); + } } else { - try { - Integer.valueOf(port); - } catch (Exception e) { - throw new DdlException("Port of Odbc table must be a number." - + "Please add properties('port'='3306') when create table"); + // Set up + host = properties.get(ODBC_HOST); + if (Strings.isNullOrEmpty(host)) { + throw new DdlException("Host of Odbc table is null. " + + "Please set proper resource or add properties('host'='xxx.xxx.xxx.xxx') when create table"); + } + port = properties.get(ODBC_PORT); + if (Strings.isNullOrEmpty(port)) { + // Maybe null pointer or number convert + throw new DdlException("Port of Odbc table is null. " + + "Please set odbc_catalog_resource or add properties('port'='3306') when create table"); + } else { + try { + Integer.valueOf(port); + } catch (Exception e) { + throw new DdlException("Port of Odbc table must be a number." + + "Please set odbc_catalog_resource or add properties('port'='3306') when create table"); + + } } - } - userName = properties.get(ODBC_USER); - if (Strings.isNullOrEmpty(userName)) { - throw new DdlException("User of Odbc table is null. " - + "Please add properties('user'='root') when create table"); - } + userName = properties.get(ODBC_USER); + if (Strings.isNullOrEmpty(userName)) { + throw new DdlException("User of Odbc table is null. " + + "Please set odbc_catalog_resource or add properties('user'='root') when create table"); + } + + passwd = properties.get(ODBC_PASSWORD); + if (passwd == null) { + throw new DdlException("Password of Odbc table is null. " + + "Please set odbc_catalog_resource or add properties('password'='xxxx') when create table"); + } - passwd = properties.get(ODBC_PASSWORD); - if (passwd == null) { - throw new DdlException("Password of Odbc table is null. " - + "Please add properties('password'='xxxx') when create table"); + driver = properties.get(ODBC_DRIVER); + if (Strings.isNullOrEmpty(driver)) { + throw new DdlException("Driver of Odbc table is null. " + + "Please set odbc_catalog_resource or add properties('diver'='xxxx') when create table"); + } + + String tableType = properties.get(ODBC_TYPE); + if (Strings.isNullOrEmpty(tableType)) { + throw new DdlException("Type of Odbc table is null. " + + "Please set odbc_catalog_resource or add properties('odbc_type'='xxxx') when create table"); + } else { + odbcTableTypeName = tableType.toLowerCase(); + if (!TABLE_TYPE_MAP.containsKey(odbcTableTypeName)) { + throw new DdlException("Invaild Odbc table type:" + tableType + + " Now Odbc table type only support:" + supportTableType()); + } + } } odbcDatabaseName = properties.get(ODBC_DATABASE); @@ -132,40 +174,52 @@ private void validate(Map properties) throws DdlException { throw new DdlException("Database of Odbc table is null. " + "Please add properties('table'='xxxx') when create table"); } + } - driver = properties.get(ODBC_DRIVER); - if (Strings.isNullOrEmpty(driver)) { - throw new DdlException("Driver of Odbc table is null. " - + "Please add properties('diver'='xxxx') when create table"); + private String getPropertyFromResource(String propertyName) { + OdbcCatalogResource odbcCatalogResource = (OdbcCatalogResource) + (Catalog.getCurrentCatalog().getResourceMgr().getResource(odbcCatalogResourceName)); + if (odbcCatalogResource == null) { + throw new RuntimeException("Resource does not exist. name: " + odbcCatalogResourceName); } - String tableType = properties.get(ODBC_TYPE); - if (Strings.isNullOrEmpty(tableType)) { - throw new DdlException("Type of Odbc table is null. " - + "Please add properties('type'='xxxx') when create table"); - } else { - odbcTableTypeName = tableType.toLowerCase(); - if (!TABLE_TYPE_MAP.containsKey(odbcTableTypeName)) { - throw new DdlException("Invaild Odbc table type:" + tableType - + " Now Odbc table type only support:" + supportTableType()); - } + String property = odbcCatalogResource.getProperties(propertyName); + if (property == null) { + throw new RuntimeException("The property:" + propertyName + " do not set in resource " + odbcCatalogResourceName); } + return property; + } + + public String getOdbcCatalogResourceName() { + return odbcCatalogResourceName; } public String getHost() { - return host; + if (host != null) { + return host; + } + return getPropertyFromResource(ODBC_HOST); } public String getPort() { - return port; + if (port != null) { + return port; + } + return getPropertyFromResource(ODBC_PORT); } public String getUserName() { - return userName; + if (userName != null) { + return userName; + } + return getPropertyFromResource(ODBC_USER); } public String getPasswd() { - return passwd; + if (passwd != null) { + return passwd; + } + return getPropertyFromResource(ODBC_PASSWORD); } public String getOdbcDatabaseName() { @@ -177,27 +231,33 @@ public String getOdbcTableName() { } public String getOdbcDriver() { - return driver; + if (driver != null) { + return driver; + } + return getPropertyFromResource(ODBC_DRIVER); } public String getOdbcTableTypeName() { - return odbcTableTypeName; + if (odbcTableTypeName != null) { + return odbcTableTypeName; + } + return getPropertyFromResource(ODBC_TYPE); } public TOdbcTableType getOdbcTableType() { - return TABLE_TYPE_MAP.get(odbcTableTypeName); + return TABLE_TYPE_MAP.get(getOdbcTableTypeName()); } public TTableDescriptor toThrift() { TOdbcTable tOdbcTable = new TOdbcTable(); - tOdbcTable.setHost(host); - tOdbcTable.setPort(port); - tOdbcTable.setUser(userName); - tOdbcTable.setPasswd(passwd); - tOdbcTable.setDb(odbcDatabaseName); - tOdbcTable.setTable(odbcTableName); - tOdbcTable.setDriver(driver); + tOdbcTable.setHost(getHost()); + tOdbcTable.setPort(getPort()); + tOdbcTable.setUser(getUserName()); + tOdbcTable.setPasswd(getPasswd()); + tOdbcTable.setDb(getOdbcDatabaseName()); + tOdbcTable.setTable(getOdbcTableName()); + tOdbcTable.setDriver(getOdbcDriver()); tOdbcTable.setType(getOdbcTableType()); TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.ODBC_TABLE, @@ -213,6 +273,8 @@ public int getSignature(int signatureVersion) { String charsetName = "UTF-8"; try { + // resource name + adler32.update(odbcCatalogResourceName.getBytes(charsetName)); // name adler32.update(name.getBytes(charsetName)); // type @@ -246,6 +308,8 @@ public void write(DataOutput out) throws IOException { super.write(out); Map serializeMap = Maps.newHashMap(); + + serializeMap.put(ODBC_CATALOG_RESOURCE, odbcCatalogResourceName); serializeMap.put(ODBC_HOST, host); serializeMap.put(ODBC_PORT, port); serializeMap.put(ODBC_USER, userName); @@ -255,11 +319,16 @@ public void write(DataOutput out) throws IOException { serializeMap.put(ODBC_DRIVER, driver); serializeMap.put(ODBC_TYPE, odbcTableTypeName); - int size = serializeMap.size(); + int size = (int) serializeMap.values().stream().filter(v -> { + return v != null; + }).count(); out.writeInt(size); + for (Map.Entry kv : serializeMap.entrySet()) { - Text.writeString(out, kv.getKey()); - Text.writeString(out, kv.getValue()); + if (kv.getValue() != null) { + Text.writeString(out, kv.getKey()); + Text.writeString(out, kv.getValue()); + } } } @@ -275,6 +344,7 @@ public void readFields(DataInput in) throws IOException { serializeMap.put(key, value); } + odbcCatalogResourceName = serializeMap.get(ODBC_CATALOG_RESOURCE); host = serializeMap.get(ODBC_HOST); port = serializeMap.get(ODBC_PORT); userName = serializeMap.get(ODBC_USER); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index e140b9ff11d3bf..fed15e6fa672c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -34,7 +34,8 @@ public abstract class Resource implements Writable { public enum ResourceType { UNKNOWN, - SPARK; + SPARK, + ODBC_CATALOG; public static ResourceType fromString(String resourceType) { for (ResourceType type : ResourceType.values()) { @@ -63,6 +64,9 @@ public static Resource fromStmt(CreateResourceStmt stmt) throws DdlException { case SPARK: resource = new SparkResource(stmt.getResourceName()); break; + case ODBC_CATALOG: + resource = new OdbcCatalogResource(stmt.getResourceName()); + break; default: throw new DdlException("Only support Spark resource."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java index e8aa5d17c541c0..d112b28ad42cb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java @@ -67,8 +67,8 @@ public ResourceMgr() { } public void createResource(CreateResourceStmt stmt) throws DdlException { - if (stmt.getResourceType() != ResourceType.SPARK) { - throw new DdlException("Only support Spark Resource."); + if (stmt.getResourceType() != ResourceType.SPARK && stmt.getResourceType() != ResourceType.ODBC_CATALOG) { + throw new DdlException("Only support SPARK and ODBC_CATALOG resource."); } String resourceName = stmt.getResourceName(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java index db3f0513c545e2..5f22b9ff9e277e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -194,6 +194,8 @@ public final class FeMetaVersion { public static final int VERSION_90 = 90; // sparkLoadAppHandle public static final int VERSION_91 = 91; + // for mysql external table support resource + public static final int VERSION_92 = 92; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_91; + public static final int VERSION_CURRENT = VERSION_92; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index e455448cd40953..575385b413150c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -21,6 +21,7 @@ import org.apache.doris.alter.RollupJobV2; import org.apache.doris.alter.SchemaChangeJobV2; import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.OdbcCatalogResource; import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.catalog.RandomDistributionInfo; import org.apache.doris.catalog.Resource; @@ -96,7 +97,8 @@ public class GsonUtils { // runtime adapter for class "Resource" private static RuntimeTypeAdapterFactory resourceTypeAdapterFactory = RuntimeTypeAdapterFactory .of(Resource.class, "clazz") - .registerSubtype(SparkResource.class, SparkResource.class.getSimpleName()); + .registerSubtype(SparkResource.class, SparkResource.class.getSimpleName()) + .registerSubtype(OdbcCatalogResource.class, OdbcCatalogResource.class.getSimpleName()); // runtime adapter for class "AlterJobV2" private static RuntimeTypeAdapterFactory alterJobV2TypeAdapterFactory = RuntimeTypeAdapterFactory diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java index f26797c5230efe..5fc91f73cb67b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java @@ -37,12 +37,14 @@ public class CreateResourceStmtTest { private Analyzer analyzer; - private String resourceName; + private String resourceName1; + private String resourceName2; @Before() public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); - resourceName = "spark0"; + resourceName1 = "spark0"; + resourceName2 = "odbc"; } @Test @@ -58,11 +60,20 @@ public void testNormal(@Mocked Catalog catalog, @Injectable PaloAuth auth) throw Map properties = Maps.newHashMap(); properties.put("type", "spark"); - CreateResourceStmt stmt = new CreateResourceStmt(true, resourceName, properties); + CreateResourceStmt stmt = new CreateResourceStmt(true, resourceName1, properties); stmt.analyze(analyzer); - Assert.assertEquals(resourceName, stmt.getResourceName()); + Assert.assertEquals(resourceName1, stmt.getResourceName()); Assert.assertEquals(Resource.ResourceType.SPARK, stmt.getResourceType()); Assert.assertEquals("CREATE EXTERNAL RESOURCE 'spark0' PROPERTIES(\"type\" = \"spark\")", stmt.toSql()); + + properties = Maps.newHashMap(); + properties.put("type", "odbc_catalog"); + stmt = new CreateResourceStmt(true, resourceName2, properties); + stmt.analyze(analyzer); + Assert.assertEquals(resourceName2, stmt.getResourceName()); + Assert.assertEquals(Resource.ResourceType.ODBC_CATALOG, stmt.getResourceType()); + Assert.assertEquals("CREATE EXTERNAL RESOURCE 'odbc' PROPERTIES(\"type\" = \"odbc_catalog\")", stmt.toSql()); + } @Test(expected = AnalysisException.class) @@ -78,7 +89,7 @@ public void testUnsupportedResourceType(@Mocked Catalog catalog, @Injectable Pal Map properties = Maps.newHashMap(); properties.put("type", "hadoop"); - CreateResourceStmt stmt = new CreateResourceStmt(true, resourceName, properties); + CreateResourceStmt stmt = new CreateResourceStmt(true, resourceName1, properties); stmt.analyze(analyzer); } } \ No newline at end of file diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java new file mode 100644 index 00000000000000..9f21f7f1df9fa0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java @@ -0,0 +1,159 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.AccessTestUtil; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateResourceStmt; +import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.UserException; +import org.apache.doris.common.proc.BaseProcResult; +import org.apache.doris.meta.MetaContext; +import org.apache.doris.mysql.privilege.PaloAuth; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.persist.DropInfo; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.Maps; +import mockit.Expectations; +import mockit.Injectable; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.HashMap; +import java.util.Map; + +public class OdbcCatalogResourceTest { + private String name; + private String type; + + private String host; + private String port; + private String user; + private String passwd; + private Map properties; + private Analyzer analyzer; + + @Before + public void setUp() { + name = "odbc"; + type = "odbc_catalog"; + host = "127.0.0.1"; + port = "7777"; + user = "doris"; + passwd = "doris"; + properties = Maps.newHashMap(); + properties.put("type", type); + properties.put("host", host); + properties.put("port", port); + properties.put("user", user); + properties.put("password", passwd); + analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + } + + @Test + public void testFromStmt(@Mocked Catalog catalog, @Injectable PaloAuth auth) + throws UserException { + new Expectations() { + { + catalog.getAuth(); + result = auth; + auth.checkGlobalPriv((ConnectContext) any, PrivPredicate.ADMIN); + result = true; + } + }; + + // host: 127.0.0.1, port: 7777, without driver and odbc_type + CreateResourceStmt stmt = new CreateResourceStmt(true, name, properties); + stmt.analyze(analyzer); + OdbcCatalogResource resource = (OdbcCatalogResource) Resource.fromStmt(stmt); + Assert.assertEquals(name, resource.getName()); + Assert.assertEquals(type, resource.getType().name().toLowerCase()); + Assert.assertEquals(host, resource.getProperties("host")); + Assert.assertEquals(port, resource.getProperties("port")); + Assert.assertEquals(user, resource.getProperties("user")); + Assert.assertEquals(passwd, resource.getProperties("password")); + + // with driver and odbc_type + properties.put("driver", "mysql"); + properties.put("odbc_type", "mysql"); + stmt = new CreateResourceStmt(true, name, properties); + stmt.analyze(analyzer); + resource = (OdbcCatalogResource) Resource.fromStmt(stmt); + Assert.assertEquals("mysql", resource.getProperties("driver")); + Assert.assertEquals("mysql", resource.getProperties("odbc_type")); + + // test getProcNodeData + BaseProcResult result = new BaseProcResult(); + resource.getProcNodeData(result); + Assert.assertEquals(7, result.getRows().size()); + } + + @Test + public void testSerialization() throws Exception { + MetaContext metaContext = new MetaContext(); + metaContext.setMetaVersion(FeMetaVersion.VERSION_92); + metaContext.setThreadLocalInfo(); + + // 1. Write objects to file + File file = new File("./odbcCatalogResource"); + file.createNewFile(); + DataOutputStream dos = new DataOutputStream(new FileOutputStream(file)); + + OdbcCatalogResource odbcCatalogResource1 = new OdbcCatalogResource("odbc1"); + odbcCatalogResource1.write(dos); + + Map configs = new HashMap<>(); + configs.put("host", "host"); + configs.put("port", "port"); + configs.put("user", "user"); + configs.put("password", "password"); + OdbcCatalogResource odbcCatalogResource2 = new OdbcCatalogResource("odbc2"); + odbcCatalogResource2.setProperties(configs); + odbcCatalogResource2.write(dos); + + dos.flush(); + dos.close(); + + // 2. Read objects from file + DataInputStream dis = new DataInputStream(new FileInputStream(file)); + + OdbcCatalogResource rOdbcCatalogResource1 = (OdbcCatalogResource) OdbcCatalogResource.read(dis); + OdbcCatalogResource rOdbcCatalogResource2 = (OdbcCatalogResource) OdbcCatalogResource.read(dis); + + Assert.assertEquals("odbc1", rOdbcCatalogResource1.getName()); + Assert.assertEquals("odbc2", rOdbcCatalogResource2.getName()); + + Assert.assertEquals(rOdbcCatalogResource2.getProperties("host"), "host"); + Assert.assertEquals(rOdbcCatalogResource2.getProperties("port"), "port"); + Assert.assertEquals(rOdbcCatalogResource2.getProperties("user"), "user"); + Assert.assertEquals(rOdbcCatalogResource2.getProperties("password"), "password"); + + // 3. delete files + dis.close(); + file.delete(); + } +} \ No newline at end of file diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index ffc0860b1f65ec..058da4b2dd1182 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -346,7 +346,7 @@ public static void beforeClass() throws Exception { "\"database\" = \"db1\",\n" + "\"table\" = \"tbl1\",\n" + "\"driver\" = \"Oracle Driver\",\n" + - "\"type\" = \"oracle\"\n" + + "\"odbc_type\" = \"oracle\"\n" + ");"); createTable("create external table test.odbc_mysql\n" + @@ -360,7 +360,7 @@ public static void beforeClass() throws Exception { "\"database\" = \"db1\",\n" + "\"table\" = \"tbl1\",\n" + "\"driver\" = \"Oracle Driver\",\n" + - "\"type\" = \"mysql\"\n" + + "\"odbc_type\" = \"mysql\"\n" + ");"); }