From 270756a49bcd67c603614d3d7846a2084a0ed223 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Tue, 8 Sep 2020 14:45:24 +0800 Subject: [PATCH 1/3] [NewFeature] Support ExternalCatalogResource to simplify external table operation. Fix #4556 1. Add new Resource ExternalCatalogResource ``` create external resource "odbc" properties ( "type" = "external_catalog", (required) "user" = "test",(required) "password" = "", (required) "host" = "192.168.0.1", (required) "port" = "8086", (required) "type" = "oracle" , (optinal,only odbc exteranl table use) "driver" = "Oracle 19 ODBC driver" (optional,only odbc exteranl table use) ) ``` 2.After create ExternalCatalogResource, can create external table like: ``` CREATE TABLE `test_mysql` ( `k1` tinyint(4) NOT NULL, `k2` smallint(6) NOT NULL, `k3` int(11) NOT NULL, `k4` bigint(20) NOT NULL, `k5` decimal(9,3) NOT NULL, `k6` char(5) NOT NULL, `k10` date DEFAULT NULL, `k11` datetime DEFAULT NULL, `k7` varchar(20) NOT NULL, `k8` double NOT NULL, `k9` float NOT NULL ) ENGINE=MYSQL PROPERTIES ( "external_catalog_resource" = "odbc", "database" = "test", "table" = "test" ); ``` --- .../Data Definition/CREATE TABLE.md | 59 ++++-- .../Data Definition/CREATE TABLE.md | 49 ++++- .../org/apache/doris/catalog/Catalog.java | 30 ++- .../catalog/ExternalCatalogResource.java | 108 ++++++++++ .../org/apache/doris/catalog/MysqlTable.java | 180 ++++++++++++----- .../org/apache/doris/catalog/OdbcTable.java | 188 ++++++++++++------ .../org/apache/doris/catalog/Resource.java | 6 +- .../org/apache/doris/catalog/ResourceMgr.java | 4 +- .../apache/doris/common/FeMetaVersion.java | 4 +- .../apache/doris/persist/gson/GsonUtils.java | 4 +- .../analysis/CreateResourceStmtTest.java | 21 +- .../catalog/ExternalCatalogResourceTest.java | 104 ++++++++++ .../apache/doris/planner/QueryPlanTest.java | 2 +- 13 files changed, 611 insertions(+), 148 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/ExternalCatalogResource.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/ExternalCatalogResourceTest.java diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index 1171adc80a7e4d..9edc40411f9caf 100644 --- a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -94,6 +94,7 @@ Syntax: * BITMAP_UNION: Only for BITMAP type Allow NULL: Default is NOT NULL. NULL value should be represented as `\N` in load source file. Notice: + The origin value of BITMAP_UNION column should be TINYINT, SMALLINT, INT, BIGINT. 2. index_definition Syntax: @@ -133,14 +134,14 @@ Syntax: "line_delimiter" = "value_delimiter" ) ``` - + ``` BROKER PROPERTIES( "username" = "name", "password" = "password" ) ``` - + For different broker, the broker properties are different Notice: Files name in "path" is separated by ",". If file name includes ",", use "%2c" instead. If file name includes "%", use "%25" instead. @@ -220,7 +221,7 @@ Syntax: ["replication_num" = "3"] ) ``` - + storage_medium: SSD or HDD, The default initial storage media can be specified by `default_storage_medium= XXX` in the fe configuration file `fe.conf`, or, if not, by default, HDD. Note: when FE configuration 'enable_strict_storage_medium_check' is' True ', if the corresponding storage medium is not set in the cluster, the construction clause 'Failed to find enough host in all backends with storage medium is SSD|HDD'. storage_cooldown_time: If storage_medium is SSD, data will be automatically moved to HDD when timeout. @@ -246,9 +247,9 @@ Syntax: "colocate_with"="table1" ) ``` - + 4) if you want to use the dynamic partitioning feature, specify it in properties - + ``` PROPERTIES ( "dynamic_partition.enable" = "true|false", @@ -268,6 +269,7 @@ Syntax: Dynamic_partition. Prefix: used to specify the partition name prefix to be created, such as the partition name prefix p, automatically creates the partition name p20200108 Dynamic_partition. Buckets: specifies the number of partition buckets that are automatically created + ``` 8. rollup_index grammar: ``` @@ -320,6 +322,7 @@ Syntax: "storage_medium" = "SSD", "storage_cooldown_time" = "2015-06-04 00:00:00" ); + ``` 3. Create an olap table, with range partitioned, distributed by hash. @@ -347,16 +350,16 @@ Syntax: "storage_medium" = "SSD", "storage_cooldown_time" = "2015-06-04 00:00:00" ); ``` - + Explain: This statement will create 3 partitions: - + ``` ( { MIN }, {"2014-01-01"} ) [ {"2014-01-01"}, {"2014-06-01"} ) [ {"2014-06-01"}, {"2014-12-01"} ) ``` - + Data outside these ranges will not be loaded. 2) Fixed Range @@ -381,8 +384,8 @@ Syntax: ); 4. Create a mysql table - - ``` + 4.1 Create MySQL table directly from external table information +``` CREATE EXTERNAL TABLE example_db.table_mysql ( k1 DATE, @@ -400,8 +403,38 @@ Syntax: "password" = "mysql_passwd", "database" = "mysql_db_test", "table" = "mysql_table_test" - ); - ``` + ) +``` + + 4.2 Create MySQL table with external catalog resource +``` + CREATE EXTERNAL RESOURCE "mysql_resource" + PROPERTIES + ( + "type" = "external_catalog", + "user" = "mysql_user", + "password" = "mysql_passwd", + "host" = "127.0.0.1", + "port" = "8239" + ); +``` +``` + CREATE EXTERNAL TABLE example_db.table_mysql + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + k4 VARCHAR(2048), + k5 DATETIME + ) + ENGINE=mysql + PROPERTIES + ( + "external_catalog_resource" = "mysql_resource", + "database" = "mysql_db_test", + "table" = "mysql_table_test" + ) +``` 5. Create a broker table, with file on HDFS, line delimit by "|", column separated by "\n" @@ -549,7 +582,7 @@ Syntax: "dynamic_partition.prefix" = "p", "dynamic_partition.buckets" = "32" ); - ``` + ``` 12. Create a table with rollup index ``` CREATE TABLE example_db.rolup_index_table diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index ca223b0de4f731..747afd0e3b4753 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -151,7 +151,7 @@ under the License. 注意： "path" 中如果有多个文件，用逗号[,]分割。如果文件名中包含逗号，那么使用 %2c 来替代。如果文件名中包含 %，使用 %25 代替 现在文件内容格式支持CSV，支持GZ，BZ2，LZ4，LZO(LZOP) 压缩格式。 - + 3) 如果是 hive，则需要在 properties 提供以下信息： ``` PROPERTIES ( @@ -159,7 +159,7 @@ under the License. "table" = "hive_table_name", "hive.metastore.uris" = "thrift://127.0.0.1:9083" ) - + ``` 其中 database 是 hive 表对应的库名字，table 是 hive 表的名字，hive.metastore.uris 是 hive metastore 服务地址。 注意：目前hive外部表仅用于Spark Load使用，不支持查询。 @@ -193,7 +193,7 @@ under the License. ... ) ``` - + 说明： 使用指定的 key 列和指定的数值范围进行分区。 1) 分区名称仅支持字母开头，字母、数字和下划线组成 @@ -202,7 +202,7 @@ under the License. 3) 分区为左闭右开区间，首个分区的左边界为做最小值 4) NULL 值只会存放在包含最小值的分区中。当包含最小值的分区被删除后，NULL 值将无法导入。 5) 可以指定一列或多列作为分区列。如果分区值缺省，则会默认填充最小值。 - + 注意： 1) 分区一般用于时间维度的数据管理 2) 有数据回溯需求的，可以考虑首个分区为空分区，以便后续增加分区 @@ -270,9 +270,9 @@ under the License. "colocate_with"="table1" ) ``` - + 4) 如果希望使用动态分区特性，需要在properties 中指定 - + ``` PROPERTIES ( "dynamic_partition.enable" = "true|false", @@ -288,7 +288,7 @@ under the License. dynamic_partition.end: 用于指定提前创建的分区数量。值必须大于0。 dynamic_partition.prefix: 用于指定创建的分区名前缀，例如分区名前缀为p，则自动创建分区名为p20200108 dynamic_partition.buckets: 用于指定自动创建的分区分桶数量 - + 5) 建表时可以批量创建多个 Rollup 语法： ``` @@ -296,7 +296,7 @@ under the License. [FROM from_index_name] [PROPERTIES ("key"="value", ...)],...) ``` - + 6) 如果希望使用 内存表 特性，需要在 properties 中指定 ``` @@ -419,6 +419,7 @@ under the License. 4. 创建一个 mysql 表 + 4.1 直接通过外表信息创建mysql表 ``` CREATE EXTERNAL TABLE example_db.table_mysql ( @@ -440,6 +441,36 @@ under the License. ) ``` + 4.2 通过External Catalog Resource创建mysql表 +``` + CREATE EXTERNAL RESOURCE "mysql_resource" + PROPERTIES + ( + "type" = "external_catalog", + "user" = "mysql_user", + "password" = "mysql_passwd", + "host" = "127.0.0.1", + "port" = "8239" + ); +``` +``` + CREATE EXTERNAL TABLE example_db.table_mysql + ( + k1 DATE, + k2 INT, + k3 SMALLINT, + k4 VARCHAR(2048), + k5 DATETIME + ) + ENGINE=mysql + PROPERTIES + ( + "external_catalog_resource" = "mysql_resource", + "database" = "mysql_db_test", + "table" = "mysql_table_test" + ) +``` + 5. 创建一个数据文件存储在HDFS上的 broker 外部表, 数据使用 "|" 分割，"\n" 换行 ``` @@ -650,3 +681,5 @@ under the License. ## keyword CREATE,TABLE + +``` \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 58a864edd0cadf..b804cc6e98e5c1 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -4059,10 +4059,14 @@ public static void getDdlStmt(Table table, List createTableStmt, List createTableStmt, List configs; + + public ExternalCatalogResource(String name) { + this(name, Maps.newHashMap()); + } + + private ExternalCatalogResource(String name, Map configs) { + super(name, ResourceType.EXTERNAL_CATALOG); + this.configs = configs; + } + + public ExternalCatalogResource getCopiedResource() { + return new ExternalCatalogResource(name, Maps.newHashMap(configs)); + } + + private void checkProperties(String propertieKey) throws DdlException { + // check the propertie key + String value = configs.get(propertieKey); + if (value == null) { + throw new DdlException("Missing " + propertieKey + " in properties"); + } + + } + + public String getProperties(String propertieKey) { + // check the propertie key + String value = configs.get(propertieKey); + return value; + } + + @Override + protected void setProperties(Map properties) throws DdlException { + Preconditions.checkState(properties != null); + + configs = properties; + + checkProperties(HOST); + checkProperties(PORT); + checkProperties(USER); + checkProperties(PASSWORD); + } + + @Override + protected void getProcNodeData(BaseProcResult result) { + String lowerCaseType = type.name().toLowerCase(); + for (Map.Entry entry : configs.entrySet()) { + result.addRow(Lists.newArrayList(name, lowerCaseType, entry.getKey(), entry.getValue())); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlTable.java index 7fa733b686d496..dd9d5447ead31f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MysqlTable.java @@ -17,8 +17,12 @@ package org.apache.doris.catalog; +import com.google.common.collect.Maps; import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TMySQLTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; @@ -39,6 +43,7 @@ public class MysqlTable extends Table { private static final Logger LOG = LogManager.getLogger(OlapTable.class); + private static final String EXTERNAL_CATALOG_RESOURCE = "external_catalog_resource"; private static final String MYSQL_HOST = "host"; private static final String MYSQL_PORT = "port"; private static final String MYSQL_USER = "user"; @@ -46,6 +51,7 @@ public class MysqlTable extends Table { private static final String MYSQL_DATABASE = "database"; private static final String MYSQL_TABLE = "table"; + private String externalCatalogResourceName; private String host; private String port; private String userName; @@ -66,43 +72,62 @@ public MysqlTable(long id, String name, List schema, Map private void validate(Map properties) throws DdlException { if (properties == null) { throw new DdlException("Please set properties of mysql table, " - + "they are: host, port, user, password, database and table"); + + "they are: external_catalog_resource or [host, port, user, password] and database and table"); } - // Set up - host = properties.get(MYSQL_HOST); - if (Strings.isNullOrEmpty(host)) { - throw new DdlException("Host of MySQL table is null. " - + "Please add properties('host'='xxx.xxx.xxx.xxx') when create table"); - } + if (properties.containsKey(EXTERNAL_CATALOG_RESOURCE)) { + externalCatalogResourceName = properties.get(EXTERNAL_CATALOG_RESOURCE); + + // 1. check whether resource exist + Resource oriResource = Catalog.getCurrentCatalog().getResourceMgr().getResource(externalCatalogResourceName); + if (oriResource == null) { + throw new DdlException("Resource does not exist. name: " + externalCatalogResourceName); + } - port = properties.get(MYSQL_PORT); - if (Strings.isNullOrEmpty(port)) { - // Maybe null pointer or number convert - throw new DdlException("Port of MySQL table is null. " - + "Please add properties('port'='3306') when create table"); + // 2. check resource usage privilege + if (!Catalog.getCurrentCatalog().getAuth().checkResourcePriv(ConnectContext.get(), + externalCatalogResourceName, + PrivPredicate.USAGE)) { + throw new DdlException("USAGE denied to user '" + ConnectContext.get().getQualifiedUser() + + "'@'" + ConnectContext.get().getRemoteIP() + + "' for resource '" + externalCatalogResourceName + "'"); + } } else { - try { - Integer.valueOf(port); - } catch (Exception e) { - throw new DdlException("Port of MySQL table must be a number." - + "Please add properties('port'='3306') when create table"); + // Set up + host = properties.get(MYSQL_HOST); + if (Strings.isNullOrEmpty(host)) { + throw new DdlException("Host of MySQL table is null. " + + "Please set proper resource or add properties('host'='xxx.xxx.xxx.xxx') when create table"); + } + port = properties.get(MYSQL_PORT); + if (Strings.isNullOrEmpty(port)) { + // Maybe null pointer or number convert + throw new DdlException("Port of MySQL table is null. " + + "Please set proper resource or add properties('port'='3306') when create table"); + } else { + try { + Integer.valueOf(port); + } catch (Exception e) { + throw new DdlException("Port of MySQL table must be a number." + + "Please set proper resource or add properties('port'='3306') when create table"); + + } } - } - userName = properties.get(MYSQL_USER); - if (Strings.isNullOrEmpty(userName)) { - throw new DdlException("User of MySQL table is null. " - + "Please add properties('user'='root') when create table"); - } + userName = properties.get(MYSQL_USER); + if (Strings.isNullOrEmpty(userName)) { + throw new DdlException("User of MySQL table is null. " + + "Please set proper resource or add properties('user'='root') when create table"); + } - passwd = properties.get(MYSQL_PASSWORD); - if (passwd == null) { - throw new DdlException("Password of MySQL table is null. " - + "Please add properties('password'='xxxx') when create table"); + passwd = properties.get(MYSQL_PASSWORD); + if (passwd == null) { + throw new DdlException("Password of MySQL table is null. " + + "Please set proper resource or add properties('password'='xxxx') when create table"); + } } - + mysqlDatabaseName = properties.get(MYSQL_DATABASE); if (Strings.isNullOrEmpty(mysqlDatabaseName)) { throw new DdlException("Database of MySQL table is null. " @@ -115,21 +140,51 @@ private void validate(Map properties) throws DdlException { + "Please add properties('table'='xxxx') when create table"); } } + + private String getPropertyFromResource(String propertyName) { + ExternalCatalogResource externalCatalogResource = (ExternalCatalogResource) + (Catalog.getCurrentCatalog().getResourceMgr().getResource(externalCatalogResourceName)); + if (externalCatalogResource == null) { + throw new RuntimeException("Resource does not exist. name: " + externalCatalogResourceName); + } + + String property = externalCatalogResource.getProperties(propertyName); + if (property == null) { + throw new RuntimeException("The property:" + propertyName + " do not set in resource " + externalCatalogResourceName); + } + return property; + } + + public String getExternalCatalogResourceName() { + return externalCatalogResourceName; + } public String getHost() { - return host; + if (host != null) { + return host; + } + return getPropertyFromResource(MYSQL_HOST); } public String getPort() { - return port; + if (port != null) { + return port; + } + return getPropertyFromResource(MYSQL_PORT); } public String getUserName() { - return userName; + if (userName != null) { + return userName; + } + return getPropertyFromResource(MYSQL_USER); } public String getPasswd() { - return passwd; + if (passwd != null) { + return passwd; + } + return getPropertyFromResource(MYSQL_PASSWORD); } public String getMysqlDatabaseName() { @@ -142,7 +197,7 @@ public String getMysqlTableName() { public TTableDescriptor toThrift() { TMySQLTable tMySQLTable = - new TMySQLTable(host, port, userName, passwd, mysqlDatabaseName, mysqlTableName); + new TMySQLTable(getHost(), getPort(), getUserName(), getPasswd(), mysqlDatabaseName, mysqlTableName); TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.MYSQL_TABLE, fullSchema.size(), 0, getName(), ""); tTableDescriptor.setMysqlTable(tMySQLTable); @@ -156,6 +211,8 @@ public int getSignature(int signatureVersion) { String charsetName = "UTF-8"; try { + // resource name + adler32.update(externalCatalogResourceName.getBytes(charsetName)); // name adler32.update(name.getBytes(charsetName)); // type @@ -185,23 +242,54 @@ public int getSignature(int signatureVersion) { public void write(DataOutput out) throws IOException { super.write(out); - Text.writeString(out, host); - Text.writeString(out, port); - Text.writeString(out, userName); - Text.writeString(out, passwd); - Text.writeString(out, mysqlDatabaseName); - Text.writeString(out, mysqlTableName); + Map serializeMap = Maps.newHashMap(); + serializeMap.put(EXTERNAL_CATALOG_RESOURCE, externalCatalogResourceName); + serializeMap.put(MYSQL_HOST, host); + serializeMap.put(MYSQL_PORT, port); + serializeMap.put(MYSQL_USER, userName); + serializeMap.put(MYSQL_PASSWORD, passwd); + serializeMap.put(MYSQL_DATABASE, mysqlDatabaseName); + serializeMap.put(MYSQL_TABLE, mysqlTableName); + + int size = (int) serializeMap.values().stream().filter(v -> { + return v != null; + }).count(); + out.writeInt(size); + for (Map.Entry kv : serializeMap.entrySet()) { + if (kv.getValue() != null) { + Text.writeString(out, kv.getKey()); + Text.writeString(out, kv.getValue()); + } + } } public void readFields(DataInput in) throws IOException { super.readFields(in); - // Read MySQL meta - host = Text.readString(in); - port = Text.readString(in); - userName = Text.readString(in); - passwd = Text.readString(in); - mysqlDatabaseName = Text.readString(in); - mysqlTableName = Text.readString(in); + if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_92) { + // Read MySQL meta + int size = in.readInt(); + Map serializeMap = Maps.newHashMap(); + for (int i = 0; i < size; i++) { + String key = Text.readString(in); + String value = Text.readString(in); + serializeMap.put(key, value); + } + + externalCatalogResourceName = serializeMap.get(EXTERNAL_CATALOG_RESOURCE); + host = serializeMap.get(MYSQL_HOST); + port = serializeMap.get(MYSQL_PORT); + userName = serializeMap.get(MYSQL_USER); + passwd = serializeMap.get(MYSQL_PASSWORD); + mysqlDatabaseName = serializeMap.get(MYSQL_DATABASE); + mysqlTableName = serializeMap.get(MYSQL_TABLE); + } else { + host = Text.readString(in); + port = Text.readString(in); + userName = Text.readString(in); + passwd = Text.readString(in); + mysqlDatabaseName = Text.readString(in); + mysqlTableName = Text.readString(in); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java index dfd9a6749c7f3d..35eb2942abb701 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java @@ -20,6 +20,8 @@ import com.google.common.collect.Maps; import org.apache.doris.common.DdlException; import org.apache.doris.common.io.Text; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TOdbcTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; @@ -43,6 +45,7 @@ public class OdbcTable extends Table { private static final Logger LOG = LogManager.getLogger(OlapTable.class); + private static final String EXTERNAL_CATALOG_RESOURCE = "external_catalog_resource"; private static final String ODBC_HOST = "host"; private static final String ODBC_PORT = "port"; private static final String ODBC_USER = "user"; @@ -50,7 +53,7 @@ public class OdbcTable extends Table { private static final String ODBC_DATABASE = "database"; private static final String ODBC_TABLE = "table"; private static final String ODBC_DRIVER = "driver"; - private static final String ODBC_TYPE = "type"; + private static final String ODBC_TYPE = "odbc_type"; private static Map TABLE_TYPE_MAP; static { @@ -62,6 +65,7 @@ public class OdbcTable extends Table { TABLE_TYPE_MAP = Collections.unmodifiableMap(tempMap); } + private String externalCatalogResourceName; private String host; private String port; private String userName; @@ -84,41 +88,79 @@ public OdbcTable(long id, String name, List schema, Map private void validate(Map properties) throws DdlException { if (properties == null) { throw new DdlException("Please set properties of odbc table, " - + "they are: host, port, user, password, database and table"); + + "they are: external_catalog_resource or [host, port, user, password, driver, odbc_type]" + + " and database and table"); } - // Set up - host = properties.get(ODBC_HOST); - if (Strings.isNullOrEmpty(host)) { - throw new DdlException("Host of Odbc table is null. " - + "Please add properties('host'='xxx.xxx.xxx.xxx') when create table"); - } + if (properties.containsKey(EXTERNAL_CATALOG_RESOURCE)) { + externalCatalogResourceName = properties.get(EXTERNAL_CATALOG_RESOURCE); + + // 1. check whether resource exist + Resource oriResource = Catalog.getCurrentCatalog().getResourceMgr().getResource(externalCatalogResourceName); + if (oriResource == null) { + throw new DdlException("Resource does not exist. name: " + externalCatalogResourceName); + } - port = properties.get(ODBC_PORT); - if (Strings.isNullOrEmpty(port)) { - // Maybe null pointer or number convert - throw new DdlException("Port of Odbc table is null. " - + "Please add properties('port'='3306') when create table"); + // 2. check resource usage privilege + if (!Catalog.getCurrentCatalog().getAuth().checkResourcePriv(ConnectContext.get(), + externalCatalogResourceName, + PrivPredicate.USAGE)) { + throw new DdlException("USAGE denied to user '" + ConnectContext.get().getQualifiedUser() + + "'@'" + ConnectContext.get().getRemoteIP() + + "' for resource '" + externalCatalogResourceName + "'"); + } } else { - try { - Integer.valueOf(port); - } catch (Exception e) { - throw new DdlException("Port of Odbc table must be a number." - + "Please add properties('port'='3306') when create table"); + // Set up + host = properties.get(ODBC_HOST); + if (Strings.isNullOrEmpty(host)) { + throw new DdlException("Host of Odbc table is null. " + + "Please set proper resource or add properties('host'='xxx.xxx.xxx.xxx') when create table"); + } + port = properties.get(ODBC_PORT); + if (Strings.isNullOrEmpty(port)) { + // Maybe null pointer or number convert + throw new DdlException("Port of Odbc table is null. " + + "Please set external_catalog_resource or add properties('port'='3306') when create table"); + } else { + try { + Integer.valueOf(port); + } catch (Exception e) { + throw new DdlException("Port of Odbc table must be a number." + + "Please set external_catalog_resource or add properties('port'='3306') when create table"); + + } } - } - userName = properties.get(ODBC_USER); - if (Strings.isNullOrEmpty(userName)) { - throw new DdlException("User of Odbc table is null. " - + "Please add properties('user'='root') when create table"); - } + userName = properties.get(ODBC_USER); + if (Strings.isNullOrEmpty(userName)) { + throw new DdlException("User of Odbc table is null. " + + "Please set external_catalog_resource or add properties('user'='root') when create table"); + } + + passwd = properties.get(ODBC_PASSWORD); + if (passwd == null) { + throw new DdlException("Password of Odbc table is null. " + + "Please set external_catalog_resource or add properties('password'='xxxx') when create table"); + } - passwd = properties.get(ODBC_PASSWORD); - if (passwd == null) { - throw new DdlException("Password of Odbc table is null. " - + "Please add properties('password'='xxxx') when create table"); + driver = properties.get(ODBC_DRIVER); + if (Strings.isNullOrEmpty(driver)) { + throw new DdlException("Driver of Odbc table is null. " + + "Please set external_catalog_resource or add properties('diver'='xxxx') when create table"); + } + + String tableType = properties.get(ODBC_TYPE); + if (Strings.isNullOrEmpty(tableType)) { + throw new DdlException("Type of Odbc table is null. " + + "Please set external_catalog_resource or add properties('odbc_type'='xxxx') when create table"); + } else { + odbcTableTypeName = tableType.toLowerCase(); + if (!TABLE_TYPE_MAP.containsKey(odbcTableTypeName)) { + throw new DdlException("Invaild Odbc table type:" + tableType + + " Now Odbc table type only support:" + supportTableType()); + } + } } odbcDatabaseName = properties.get(ODBC_DATABASE); @@ -132,40 +174,52 @@ private void validate(Map properties) throws DdlException { throw new DdlException("Database of Odbc table is null. " + "Please add properties('table'='xxxx') when create table"); } + } - driver = properties.get(ODBC_DRIVER); - if (Strings.isNullOrEmpty(driver)) { - throw new DdlException("Driver of Odbc table is null. " - + "Please add properties('diver'='xxxx') when create table"); + private String getPropertyFromResource(String propertyName) { + ExternalCatalogResource externalCatalogResource = (ExternalCatalogResource) + (Catalog.getCurrentCatalog().getResourceMgr().getResource(externalCatalogResourceName)); + if (externalCatalogResource == null) { + throw new RuntimeException("Resource does not exist. name: " + externalCatalogResourceName); } - String tableType = properties.get(ODBC_TYPE); - if (Strings.isNullOrEmpty(tableType)) { - throw new DdlException("Type of Odbc table is null. " - + "Please add properties('type'='xxxx') when create table"); - } else { - odbcTableTypeName = tableType.toLowerCase(); - if (!TABLE_TYPE_MAP.containsKey(odbcTableTypeName)) { - throw new DdlException("Invaild Odbc table type:" + tableType - + " Now Odbc table type only support:" + supportTableType()); - } + String property = externalCatalogResource.getProperties(propertyName); + if (property == null) { + throw new RuntimeException("The property:" + propertyName + " do not set in resource " + externalCatalogResourceName); } + return property; + } + + public String getExternalCatalogResourceName() { + return externalCatalogResourceName; } public String getHost() { - return host; + if (host != null) { + return host; + } + return getPropertyFromResource(ODBC_HOST); } public String getPort() { - return port; + if (port != null) { + return port; + } + return getPropertyFromResource(ODBC_PORT); } public String getUserName() { - return userName; + if (userName != null) { + return userName; + } + return getPropertyFromResource(ODBC_USER); } public String getPasswd() { - return passwd; + if (passwd != null) { + return passwd; + } + return getPropertyFromResource(ODBC_PASSWORD); } public String getOdbcDatabaseName() { @@ -177,27 +231,33 @@ public String getOdbcTableName() { } public String getOdbcDriver() { - return driver; + if (driver != null) { + return driver; + } + return getPropertyFromResource(ODBC_DRIVER); } public String getOdbcTableTypeName() { - return odbcTableTypeName; + if (odbcTableTypeName != null) { + return odbcTableTypeName; + } + return getPropertyFromResource(ODBC_TYPE); } public TOdbcTableType getOdbcTableType() { - return TABLE_TYPE_MAP.get(odbcTableTypeName); + return TABLE_TYPE_MAP.get(getOdbcTableTypeName()); } public TTableDescriptor toThrift() { TOdbcTable tOdbcTable = new TOdbcTable(); - tOdbcTable.setHost(host); - tOdbcTable.setPort(port); - tOdbcTable.setUser(userName); - tOdbcTable.setPasswd(passwd); - tOdbcTable.setDb(odbcDatabaseName); - tOdbcTable.setTable(odbcTableName); - tOdbcTable.setDriver(driver); + tOdbcTable.setHost(getHost()); + tOdbcTable.setPort(getPort()); + tOdbcTable.setUser(getUserName()); + tOdbcTable.setPasswd(getPasswd()); + tOdbcTable.setDb(getOdbcDatabaseName()); + tOdbcTable.setTable(getOdbcTableName()); + tOdbcTable.setDriver(getOdbcDriver()); tOdbcTable.setType(getOdbcTableType()); TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.ODBC_TABLE, @@ -213,6 +273,8 @@ public int getSignature(int signatureVersion) { String charsetName = "UTF-8"; try { + // resource name + adler32.update(externalCatalogResourceName.getBytes(charsetName)); // name adler32.update(name.getBytes(charsetName)); // type @@ -246,6 +308,8 @@ public void write(DataOutput out) throws IOException { super.write(out); Map serializeMap = Maps.newHashMap(); + + serializeMap.put(EXTERNAL_CATALOG_RESOURCE, externalCatalogResourceName); serializeMap.put(ODBC_HOST, host); serializeMap.put(ODBC_PORT, port); serializeMap.put(ODBC_USER, userName); @@ -255,11 +319,16 @@ public void write(DataOutput out) throws IOException { serializeMap.put(ODBC_DRIVER, driver); serializeMap.put(ODBC_TYPE, odbcTableTypeName); - int size = serializeMap.size(); + int size = (int) serializeMap.values().stream().filter(v -> { + return v != null; + }).count(); out.writeInt(size); + for (Map.Entry kv : serializeMap.entrySet()) { - Text.writeString(out, kv.getKey()); - Text.writeString(out, kv.getValue()); + if (kv.getValue() != null) { + Text.writeString(out, kv.getKey()); + Text.writeString(out, kv.getValue()); + } } } @@ -275,6 +344,7 @@ public void readFields(DataInput in) throws IOException { serializeMap.put(key, value); } + externalCatalogResourceName = serializeMap.get(EXTERNAL_CATALOG_RESOURCE); host = serializeMap.get(ODBC_HOST); port = serializeMap.get(ODBC_PORT); userName = serializeMap.get(ODBC_USER); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index e140b9ff11d3bf..e9bbb6735ce80e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -34,7 +34,8 @@ public abstract class Resource implements Writable { public enum ResourceType { UNKNOWN, - SPARK; + SPARK, + EXTERNAL_CATALOG; public static ResourceType fromString(String resourceType) { for (ResourceType type : ResourceType.values()) { @@ -63,6 +64,9 @@ public static Resource fromStmt(CreateResourceStmt stmt) throws DdlException { case SPARK: resource = new SparkResource(stmt.getResourceName()); break; + case EXTERNAL_CATALOG: + resource = new ExternalCatalogResource(stmt.getResourceName()); + break; default: throw new DdlException("Only support Spark resource."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java index e8aa5d17c541c0..a1bd90b686de3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java @@ -67,8 +67,8 @@ public ResourceMgr() { } public void createResource(CreateResourceStmt stmt) throws DdlException { - if (stmt.getResourceType() != ResourceType.SPARK) { - throw new DdlException("Only support Spark Resource."); + if (stmt.getResourceType() != ResourceType.SPARK && stmt.getResourceType() != ResourceType.EXTERNAL_CATALOG) { + throw new DdlException("Only support SPARK and EXTERNAL_CATALOG resource."); } String resourceName = stmt.getResourceName(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java index db3f0513c545e2..5f22b9ff9e277e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeMetaVersion.java @@ -194,6 +194,8 @@ public final class FeMetaVersion { public static final int VERSION_90 = 90; // sparkLoadAppHandle public static final int VERSION_91 = 91; + // for mysql external table support resource + public static final int VERSION_92 = 92; // note: when increment meta version, should assign the latest version to VERSION_CURRENT - public static final int VERSION_CURRENT = VERSION_91; + public static final int VERSION_CURRENT = VERSION_92; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index e455448cd40953..ad756adda1382f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -21,6 +21,7 @@ import org.apache.doris.alter.RollupJobV2; import org.apache.doris.alter.SchemaChangeJobV2; import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.ExternalCatalogResource; import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.catalog.RandomDistributionInfo; import org.apache.doris.catalog.Resource; @@ -96,7 +97,8 @@ public class GsonUtils { // runtime adapter for class "Resource" private static RuntimeTypeAdapterFactory resourceTypeAdapterFactory = RuntimeTypeAdapterFactory .of(Resource.class, "clazz") - .registerSubtype(SparkResource.class, SparkResource.class.getSimpleName()); + .registerSubtype(SparkResource.class, SparkResource.class.getSimpleName()) + .registerSubtype(ExternalCatalogResource.class, ExternalCatalogResource.class.getSimpleName()); // runtime adapter for class "AlterJobV2" private static RuntimeTypeAdapterFactory alterJobV2TypeAdapterFactory = RuntimeTypeAdapterFactory diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java index f26797c5230efe..c6dd0e56816e8d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java @@ -37,12 +37,14 @@ public class CreateResourceStmtTest { private Analyzer analyzer; - private String resourceName; + private String resourceName1; + private String resourceName2; @Before() public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); - resourceName = "spark0"; + resourceName1 = "spark0"; + resourceName2 = "odbc"; } @Test @@ -58,11 +60,20 @@ public void testNormal(@Mocked Catalog catalog, @Injectable PaloAuth auth) throw Map properties = Maps.newHashMap(); properties.put("type", "spark"); - CreateResourceStmt stmt = new CreateResourceStmt(true, resourceName, properties); + CreateResourceStmt stmt = new CreateResourceStmt(true, resourceName1, properties); stmt.analyze(analyzer); - Assert.assertEquals(resourceName, stmt.getResourceName()); + Assert.assertEquals(resourceName1, stmt.getResourceName()); Assert.assertEquals(Resource.ResourceType.SPARK, stmt.getResourceType()); Assert.assertEquals("CREATE EXTERNAL RESOURCE 'spark0' PROPERTIES(\"type\" = \"spark\")", stmt.toSql()); + + properties = Maps.newHashMap(); + properties.put("type", "external_catalog"); + stmt = new CreateResourceStmt(true, resourceName2, properties); + stmt.analyze(analyzer); + Assert.assertEquals(resourceName2, stmt.getResourceName()); + Assert.assertEquals(Resource.ResourceType.EXTERNAL_CATALOG, stmt.getResourceType()); + Assert.assertEquals("CREATE EXTERNAL RESOURCE 'odbc' PROPERTIES(\"type\" = \"external_catalog\")", stmt.toSql()); + } @Test(expected = AnalysisException.class) @@ -78,7 +89,7 @@ public void testUnsupportedResourceType(@Mocked Catalog catalog, @Injectable Pal Map properties = Maps.newHashMap(); properties.put("type", "hadoop"); - CreateResourceStmt stmt = new CreateResourceStmt(true, resourceName, properties); + CreateResourceStmt stmt = new CreateResourceStmt(true, resourceName1, properties); stmt.analyze(analyzer); } } \ No newline at end of file diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ExternalCatalogResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ExternalCatalogResourceTest.java new file mode 100644 index 00000000000000..17c6ef1a5cc648 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ExternalCatalogResourceTest.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.AccessTestUtil; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateResourceStmt; +import org.apache.doris.common.UserException; +import org.apache.doris.common.proc.BaseProcResult; +import org.apache.doris.mysql.privilege.PaloAuth; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.Maps; +import mockit.Expectations; +import mockit.Injectable; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Map; + +public class ExternalCatalogResourceTest { + private String name; + private String type; + + private String host; + private String port; + private String user; + private String passwd; + private Map properties; + private Analyzer analyzer; + + @Before + public void setUp() { + name = "odbc"; + type = "external_catalog"; + host = "127.0.0.1"; + port = "7777"; + user = "doris"; + passwd = "doris"; + properties = Maps.newHashMap(); + properties.put("type", type); + properties.put("host", host); + properties.put("port", port); + properties.put("user", user); + properties.put("password", passwd); + analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + } + + @Test + public void testFromStmt(@Mocked Catalog catalog, @Injectable PaloAuth auth) + throws UserException { + new Expectations() { + { + catalog.getAuth(); + result = auth; + auth.checkGlobalPriv((ConnectContext) any, PrivPredicate.ADMIN); + result = true; + } + }; + + // host: 127.0.0.1, port: 7777, without driver and odbc_type + CreateResourceStmt stmt = new CreateResourceStmt(true, name, properties); + stmt.analyze(analyzer); + ExternalCatalogResource resource = (ExternalCatalogResource) Resource.fromStmt(stmt); + Assert.assertEquals(name, resource.getName()); + Assert.assertEquals(type, resource.getType().name().toLowerCase()); + Assert.assertEquals(host, resource.getProperties("host")); + Assert.assertEquals(port, resource.getProperties("port")); + Assert.assertEquals(user, resource.getProperties("user")); + Assert.assertEquals(passwd, resource.getProperties("password")); + + // with driver and odbc_type + properties.put("driver", "mysql"); + properties.put("odbc_type", "mysql"); + stmt = new CreateResourceStmt(true, name, properties); + stmt.analyze(analyzer); + resource = (ExternalCatalogResource) Resource.fromStmt(stmt); + Assert.assertEquals("mysql", resource.getProperties("driver")); + Assert.assertEquals("mysql", resource.getProperties("odbc_type")); + + // test getProcNodeData + BaseProcResult result = new BaseProcResult(); + resource.getProcNodeData(result); + Assert.assertEquals(7, result.getRows().size()); + } +} \ No newline at end of file diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index ffc0860b1f65ec..670bd2a33d7684 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -346,7 +346,7 @@ public static void beforeClass() throws Exception { "\"database\" = \"db1\",\n" + "\"table\" = \"tbl1\",\n" + "\"driver\" = \"Oracle Driver\",\n" + - "\"type\" = \"oracle\"\n" + + "\"odbc_type\" = \"oracle\"\n" + ");"); createTable("create external table test.odbc_mysql\n" + From 4c94d858676bf68a734aedb44f06a6e387f95d36 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Mon, 21 Sep 2020 15:26:06 +0800 Subject: [PATCH 2/3] change ExternalCatalogResource to OdbcCatalogResource --- .../Data Definition/CREATE TABLE.md | 8 +-- .../Data Definition/CREATE TABLE.md | 6 +-- .../doris/analysis/ShowResourcesStmt.java | 2 +- .../org/apache/doris/catalog/Catalog.java | 10 ++-- .../org/apache/doris/catalog/MysqlTable.java | 48 ++++++++--------- ...Resource.java => OdbcCatalogResource.java} | 19 +++---- .../org/apache/doris/catalog/OdbcTable.java | 52 +++++++++---------- .../org/apache/doris/catalog/Resource.java | 6 +-- .../org/apache/doris/catalog/ResourceMgr.java | 4 +- .../apache/doris/persist/gson/GsonUtils.java | 4 +- .../analysis/CreateResourceStmtTest.java | 6 +-- ...Test.java => OdbcCatalogResourceTest.java} | 9 ++-- .../apache/doris/planner/QueryPlanTest.java | 2 +- 13 files changed, 88 insertions(+), 88 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/catalog/{ExternalCatalogResource.java => OdbcCatalogResource.java} (86%) rename fe/fe-core/src/test/java/org/apache/doris/catalog/{ExternalCatalogResourceTest.java => OdbcCatalogResourceTest.java} (93%) diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index 9edc40411f9caf..f8a3188328391b 100644 --- a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -406,16 +406,16 @@ Syntax: ) ``` - 4.2 Create MySQL table with external catalog resource + 4.2 Create MySQL table with external ODBC catalog resource ``` CREATE EXTERNAL RESOURCE "mysql_resource" PROPERTIES ( - "type" = "external_catalog", + "type" = "odbc_catalog", "user" = "mysql_user", "password" = "mysql_passwd", "host" = "127.0.0.1", - "port" = "8239" + "port" = "8239" ); ``` ``` @@ -430,7 +430,7 @@ Syntax: ENGINE=mysql PROPERTIES ( - "external_catalog_resource" = "mysql_resource", + "odbc_catalog_resource" = "mysql_resource", "database" = "mysql_db_test", "table" = "mysql_table_test" ) diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md index 747afd0e3b4753..f1e7c8fa32fb24 100644 --- a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md +++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md @@ -446,7 +446,7 @@ under the License. CREATE EXTERNAL RESOURCE "mysql_resource" PROPERTIES ( - "type" = "external_catalog", + "type" = "odbc_catalog", "user" = "mysql_user", "password" = "mysql_passwd", "host" = "127.0.0.1", @@ -465,7 +465,7 @@ under the License. ENGINE=mysql PROPERTIES ( - "external_catalog_resource" = "mysql_resource", + "odbc_catalog_resource" = "mysql_resource", "database" = "mysql_db_test", "table" = "mysql_table_test" ) @@ -682,4 +682,4 @@ under the License. CREATE,TABLE -``` \ No newline at end of file +``` diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java index 1a1c88df3a1aa9..edb39ffcbd22c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowResourcesStmt.java @@ -215,7 +215,7 @@ private void analyzeSubPredicate(Expr subExpr) throws AnalysisException { if (!valid) { throw new AnalysisException("Where clause should looks like: NAME = \"your_resource_name\"," - + " or NAME LIKE \"matcher\", " + " or RESOURCETYPE = \"SPARK\", " + + " or NAME LIKE \"matcher\", " + " or RESOURCETYPE = \"resource_type\", " + " or compound predicate with operator AND"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index b804cc6e98e5c1..8b47b1161171d3 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -4059,13 +4059,13 @@ public static void getDdlStmt(Table table, List createTableStmt, List createTableStmt, List schema, Map private void validate(Map properties) throws DdlException { if (properties == null) { throw new DdlException("Please set properties of mysql table, " - + "they are: external_catalog_resource or [host, port, user, password] and database and table"); + + "they are: odbc_catalog_resource or [host, port, user, password] and database and table"); } - if (properties.containsKey(EXTERNAL_CATALOG_RESOURCE)) { - externalCatalogResourceName = properties.get(EXTERNAL_CATALOG_RESOURCE); + if (properties.containsKey(ODBC_CATALOG_RESOURCE)) { + odbcCatalogResourceName = properties.get(ODBC_CATALOG_RESOURCE); // 1. check whether resource exist - Resource oriResource = Catalog.getCurrentCatalog().getResourceMgr().getResource(externalCatalogResourceName); + Resource oriResource = Catalog.getCurrentCatalog().getResourceMgr().getResource(odbcCatalogResourceName); if (oriResource == null) { - throw new DdlException("Resource does not exist. name: " + externalCatalogResourceName); + throw new DdlException("Resource does not exist. name: " + odbcCatalogResourceName); } // 2. check resource usage privilege if (!Catalog.getCurrentCatalog().getAuth().checkResourcePriv(ConnectContext.get(), - externalCatalogResourceName, + odbcCatalogResourceName, PrivPredicate.USAGE)) { throw new DdlException("USAGE denied to user '" + ConnectContext.get().getQualifiedUser() + "'@'" + ConnectContext.get().getRemoteIP() - + "' for resource '" + externalCatalogResourceName + "'"); + + "' for resource '" + odbcCatalogResourceName + "'"); } } else { // Set up @@ -142,21 +142,21 @@ private void validate(Map properties) throws DdlException { } private String getPropertyFromResource(String propertyName) { - ExternalCatalogResource externalCatalogResource = (ExternalCatalogResource) - (Catalog.getCurrentCatalog().getResourceMgr().getResource(externalCatalogResourceName)); - if (externalCatalogResource == null) { - throw new RuntimeException("Resource does not exist. name: " + externalCatalogResourceName); + OdbcCatalogResource odbcCatalogResource = (OdbcCatalogResource) + (Catalog.getCurrentCatalog().getResourceMgr().getResource(odbcCatalogResourceName)); + if (odbcCatalogResource == null) { + throw new RuntimeException("Resource does not exist. name: " + odbcCatalogResourceName); } - String property = externalCatalogResource.getProperties(propertyName); + String property = odbcCatalogResource.getProperties(propertyName); if (property == null) { - throw new RuntimeException("The property:" + propertyName + " do not set in resource " + externalCatalogResourceName); + throw new RuntimeException("The property:" + propertyName + " do not set in resource " + odbcCatalogResourceName); } return property; } - public String getExternalCatalogResourceName() { - return externalCatalogResourceName; + public String getOdbcCatalogResourceName() { + return odbcCatalogResourceName; } public String getHost() { @@ -211,20 +211,18 @@ public int getSignature(int signatureVersion) { String charsetName = "UTF-8"; try { - // resource name - adler32.update(externalCatalogResourceName.getBytes(charsetName)); // name adler32.update(name.getBytes(charsetName)); // type adler32.update(type.name().getBytes(charsetName)); // host - adler32.update(host.getBytes(charsetName)); + adler32.update(getHost().getBytes(charsetName)); // port - adler32.update(port.getBytes(charsetName)); + adler32.update(getPort().getBytes(charsetName)); // username - adler32.update(userName.getBytes(charsetName)); + adler32.update(getUserName().getBytes(charsetName)); // passwd - adler32.update(passwd.getBytes(charsetName)); + adler32.update(getPasswd().getBytes(charsetName)); // mysql db adler32.update(mysqlDatabaseName.getBytes(charsetName)); // mysql table @@ -243,7 +241,7 @@ public void write(DataOutput out) throws IOException { super.write(out); Map serializeMap = Maps.newHashMap(); - serializeMap.put(EXTERNAL_CATALOG_RESOURCE, externalCatalogResourceName); + serializeMap.put(ODBC_CATALOG_RESOURCE, odbcCatalogResourceName); serializeMap.put(MYSQL_HOST, host); serializeMap.put(MYSQL_PORT, port); serializeMap.put(MYSQL_USER, userName); @@ -276,7 +274,7 @@ public void readFields(DataInput in) throws IOException { serializeMap.put(key, value); } - externalCatalogResourceName = serializeMap.get(EXTERNAL_CATALOG_RESOURCE); + odbcCatalogResourceName = serializeMap.get(ODBC_CATALOG_RESOURCE); host = serializeMap.get(MYSQL_HOST); port = serializeMap.get(MYSQL_PORT); userName = serializeMap.get(MYSQL_USER); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ExternalCatalogResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java similarity index 86% rename from fe/fe-core/src/main/java/org/apache/doris/catalog/ExternalCatalogResource.java rename to fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java index 72491611038aa5..4bf547d5e1a8de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ExternalCatalogResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java @@ -27,13 +27,13 @@ import java.util.Map; /** - * External Catalog resource for external table query. + * External ODBC Catalog resource for external table query. * - * External Catalog resource example: + * External ODBC Catalog resource example: * CREATE EXTERNAL RESOURCE "odbc_mysql" * PROPERTIES * ( - * "type" = "external_catalog", [required] + * "type" = "external_odbc", [required] * "user" = "root", [required] * "password" = "root", [required] * "host" = "192.168.1.1", [required] @@ -44,7 +44,7 @@ * * DROP RESOURCE "odbc_mysql"; */ -public class ExternalCatalogResource extends Resource { +public class OdbcCatalogResource extends Resource { // required private static final String HOST = "host"; private static final String PORT = "port"; @@ -58,17 +58,17 @@ public class ExternalCatalogResource extends Resource { @SerializedName(value = "configs") private Map configs; - public ExternalCatalogResource(String name) { + public OdbcCatalogResource(String name) { this(name, Maps.newHashMap()); } - private ExternalCatalogResource(String name, Map configs) { - super(name, ResourceType.EXTERNAL_CATALOG); + private OdbcCatalogResource(String name, Map configs) { + super(name, ResourceType.ODBC_CATALOG); this.configs = configs; } - public ExternalCatalogResource getCopiedResource() { - return new ExternalCatalogResource(name, Maps.newHashMap(configs)); + public OdbcCatalogResource getCopiedResource() { + return new OdbcCatalogResource(name, Maps.newHashMap(configs)); } private void checkProperties(String propertieKey) throws DdlException { @@ -106,3 +106,4 @@ protected void getProcNodeData(BaseProcResult result) { } } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java index 35eb2942abb701..ee382791511a44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcTable.java @@ -45,7 +45,7 @@ public class OdbcTable extends Table { private static final Logger LOG = LogManager.getLogger(OlapTable.class); - private static final String EXTERNAL_CATALOG_RESOURCE = "external_catalog_resource"; + private static final String ODBC_CATALOG_RESOURCE = "odbc_catalog_resource"; private static final String ODBC_HOST = "host"; private static final String ODBC_PORT = "port"; private static final String ODBC_USER = "user"; @@ -65,7 +65,7 @@ public class OdbcTable extends Table { TABLE_TYPE_MAP = Collections.unmodifiableMap(tempMap); } - private String externalCatalogResourceName; + private String odbcCatalogResourceName; private String host; private String port; private String userName; @@ -88,26 +88,26 @@ public OdbcTable(long id, String name, List schema, Map private void validate(Map properties) throws DdlException { if (properties == null) { throw new DdlException("Please set properties of odbc table, " - + "they are: external_catalog_resource or [host, port, user, password, driver, odbc_type]" + + + "they are: odbc_catalog_resource or [host, port, user, password, driver, odbc_type]" + " and database and table"); } - if (properties.containsKey(EXTERNAL_CATALOG_RESOURCE)) { - externalCatalogResourceName = properties.get(EXTERNAL_CATALOG_RESOURCE); + if (properties.containsKey(ODBC_CATALOG_RESOURCE)) { + odbcCatalogResourceName = properties.get(ODBC_CATALOG_RESOURCE); // 1. check whether resource exist - Resource oriResource = Catalog.getCurrentCatalog().getResourceMgr().getResource(externalCatalogResourceName); + Resource oriResource = Catalog.getCurrentCatalog().getResourceMgr().getResource(odbcCatalogResourceName); if (oriResource == null) { - throw new DdlException("Resource does not exist. name: " + externalCatalogResourceName); + throw new DdlException("Resource does not exist. name: " + odbcCatalogResourceName); } // 2. check resource usage privilege if (!Catalog.getCurrentCatalog().getAuth().checkResourcePriv(ConnectContext.get(), - externalCatalogResourceName, + odbcCatalogResourceName, PrivPredicate.USAGE)) { throw new DdlException("USAGE denied to user '" + ConnectContext.get().getQualifiedUser() + "'@'" + ConnectContext.get().getRemoteIP() - + "' for resource '" + externalCatalogResourceName + "'"); + + "' for resource '" + odbcCatalogResourceName + "'"); } } else { // Set up @@ -121,13 +121,13 @@ private void validate(Map properties) throws DdlException { if (Strings.isNullOrEmpty(port)) { // Maybe null pointer or number convert throw new DdlException("Port of Odbc table is null. " - + "Please set external_catalog_resource or add properties('port'='3306') when create table"); + + "Please set odbc_catalog_resource or add properties('port'='3306') when create table"); } else { try { Integer.valueOf(port); } catch (Exception e) { throw new DdlException("Port of Odbc table must be a number." - + "Please set external_catalog_resource or add properties('port'='3306') when create table"); + + "Please set odbc_catalog_resource or add properties('port'='3306') when create table"); } } @@ -135,25 +135,25 @@ private void validate(Map properties) throws DdlException { userName = properties.get(ODBC_USER); if (Strings.isNullOrEmpty(userName)) { throw new DdlException("User of Odbc table is null. " - + "Please set external_catalog_resource or add properties('user'='root') when create table"); + + "Please set odbc_catalog_resource or add properties('user'='root') when create table"); } passwd = properties.get(ODBC_PASSWORD); if (passwd == null) { throw new DdlException("Password of Odbc table is null. " - + "Please set external_catalog_resource or add properties('password'='xxxx') when create table"); + + "Please set odbc_catalog_resource or add properties('password'='xxxx') when create table"); } driver = properties.get(ODBC_DRIVER); if (Strings.isNullOrEmpty(driver)) { throw new DdlException("Driver of Odbc table is null. " - + "Please set external_catalog_resource or add properties('diver'='xxxx') when create table"); + + "Please set odbc_catalog_resource or add properties('diver'='xxxx') when create table"); } String tableType = properties.get(ODBC_TYPE); if (Strings.isNullOrEmpty(tableType)) { throw new DdlException("Type of Odbc table is null. " - + "Please set external_catalog_resource or add properties('odbc_type'='xxxx') when create table"); + + "Please set odbc_catalog_resource or add properties('odbc_type'='xxxx') when create table"); } else { odbcTableTypeName = tableType.toLowerCase(); if (!TABLE_TYPE_MAP.containsKey(odbcTableTypeName)) { @@ -177,21 +177,21 @@ private void validate(Map properties) throws DdlException { } private String getPropertyFromResource(String propertyName) { - ExternalCatalogResource externalCatalogResource = (ExternalCatalogResource) - (Catalog.getCurrentCatalog().getResourceMgr().getResource(externalCatalogResourceName)); - if (externalCatalogResource == null) { - throw new RuntimeException("Resource does not exist. name: " + externalCatalogResourceName); + OdbcCatalogResource odbcCatalogResource = (OdbcCatalogResource) + (Catalog.getCurrentCatalog().getResourceMgr().getResource(odbcCatalogResourceName)); + if (odbcCatalogResource == null) { + throw new RuntimeException("Resource does not exist. name: " + odbcCatalogResourceName); } - String property = externalCatalogResource.getProperties(propertyName); + String property = odbcCatalogResource.getProperties(propertyName); if (property == null) { - throw new RuntimeException("The property:" + propertyName + " do not set in resource " + externalCatalogResourceName); + throw new RuntimeException("The property:" + propertyName + " do not set in resource " + odbcCatalogResourceName); } return property; } - public String getExternalCatalogResourceName() { - return externalCatalogResourceName; + public String getOdbcCatalogResourceName() { + return odbcCatalogResourceName; } public String getHost() { @@ -274,7 +274,7 @@ public int getSignature(int signatureVersion) { try { // resource name - adler32.update(externalCatalogResourceName.getBytes(charsetName)); + adler32.update(odbcCatalogResourceName.getBytes(charsetName)); // name adler32.update(name.getBytes(charsetName)); // type @@ -309,7 +309,7 @@ public void write(DataOutput out) throws IOException { Map serializeMap = Maps.newHashMap(); - serializeMap.put(EXTERNAL_CATALOG_RESOURCE, externalCatalogResourceName); + serializeMap.put(ODBC_CATALOG_RESOURCE, odbcCatalogResourceName); serializeMap.put(ODBC_HOST, host); serializeMap.put(ODBC_PORT, port); serializeMap.put(ODBC_USER, userName); @@ -344,7 +344,7 @@ public void readFields(DataInput in) throws IOException { serializeMap.put(key, value); } - externalCatalogResourceName = serializeMap.get(EXTERNAL_CATALOG_RESOURCE); + odbcCatalogResourceName = serializeMap.get(ODBC_CATALOG_RESOURCE); host = serializeMap.get(ODBC_HOST); port = serializeMap.get(ODBC_PORT); userName = serializeMap.get(ODBC_USER); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index e9bbb6735ce80e..fed15e6fa672c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -35,7 +35,7 @@ public abstract class Resource implements Writable { public enum ResourceType { UNKNOWN, SPARK, - EXTERNAL_CATALOG; + ODBC_CATALOG; public static ResourceType fromString(String resourceType) { for (ResourceType type : ResourceType.values()) { @@ -64,8 +64,8 @@ public static Resource fromStmt(CreateResourceStmt stmt) throws DdlException { case SPARK: resource = new SparkResource(stmt.getResourceName()); break; - case EXTERNAL_CATALOG: - resource = new ExternalCatalogResource(stmt.getResourceName()); + case ODBC_CATALOG: + resource = new OdbcCatalogResource(stmt.getResourceName()); break; default: throw new DdlException("Only support Spark resource."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java index a1bd90b686de3c..d112b28ad42cb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java @@ -67,8 +67,8 @@ public ResourceMgr() { } public void createResource(CreateResourceStmt stmt) throws DdlException { - if (stmt.getResourceType() != ResourceType.SPARK && stmt.getResourceType() != ResourceType.EXTERNAL_CATALOG) { - throw new DdlException("Only support SPARK and EXTERNAL_CATALOG resource."); + if (stmt.getResourceType() != ResourceType.SPARK && stmt.getResourceType() != ResourceType.ODBC_CATALOG) { + throw new DdlException("Only support SPARK and ODBC_CATALOG resource."); } String resourceName = stmt.getResourceName(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index ad756adda1382f..575385b413150c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -21,7 +21,7 @@ import org.apache.doris.alter.RollupJobV2; import org.apache.doris.alter.SchemaChangeJobV2; import org.apache.doris.catalog.DistributionInfo; -import org.apache.doris.catalog.ExternalCatalogResource; +import org.apache.doris.catalog.OdbcCatalogResource; import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.catalog.RandomDistributionInfo; import org.apache.doris.catalog.Resource; @@ -98,7 +98,7 @@ public class GsonUtils { private static RuntimeTypeAdapterFactory resourceTypeAdapterFactory = RuntimeTypeAdapterFactory .of(Resource.class, "clazz") .registerSubtype(SparkResource.class, SparkResource.class.getSimpleName()) - .registerSubtype(ExternalCatalogResource.class, ExternalCatalogResource.class.getSimpleName()); + .registerSubtype(OdbcCatalogResource.class, OdbcCatalogResource.class.getSimpleName()); // runtime adapter for class "AlterJobV2" private static RuntimeTypeAdapterFactory alterJobV2TypeAdapterFactory = RuntimeTypeAdapterFactory diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java index c6dd0e56816e8d..5fc91f73cb67b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateResourceStmtTest.java @@ -67,12 +67,12 @@ public void testNormal(@Mocked Catalog catalog, @Injectable PaloAuth auth) throw Assert.assertEquals("CREATE EXTERNAL RESOURCE 'spark0' PROPERTIES(\"type\" = \"spark\")", stmt.toSql()); properties = Maps.newHashMap(); - properties.put("type", "external_catalog"); + properties.put("type", "odbc_catalog"); stmt = new CreateResourceStmt(true, resourceName2, properties); stmt.analyze(analyzer); Assert.assertEquals(resourceName2, stmt.getResourceName()); - Assert.assertEquals(Resource.ResourceType.EXTERNAL_CATALOG, stmt.getResourceType()); - Assert.assertEquals("CREATE EXTERNAL RESOURCE 'odbc' PROPERTIES(\"type\" = \"external_catalog\")", stmt.toSql()); + Assert.assertEquals(Resource.ResourceType.ODBC_CATALOG, stmt.getResourceType()); + Assert.assertEquals("CREATE EXTERNAL RESOURCE 'odbc' PROPERTIES(\"type\" = \"odbc_catalog\")", stmt.toSql()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ExternalCatalogResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java similarity index 93% rename from fe/fe-core/src/test/java/org/apache/doris/catalog/ExternalCatalogResourceTest.java rename to fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java index 17c6ef1a5cc648..1b48a2417ff273 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ExternalCatalogResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java @@ -1,3 +1,4 @@ + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -36,7 +37,7 @@ import java.util.Map; -public class ExternalCatalogResourceTest { +public class OdbcCatalogResourceTest { private String name; private String type; @@ -50,7 +51,7 @@ public class ExternalCatalogResourceTest { @Before public void setUp() { name = "odbc"; - type = "external_catalog"; + type = "odbc_catalog"; host = "127.0.0.1"; port = "7777"; user = "doris"; @@ -79,7 +80,7 @@ public void testFromStmt(@Mocked Catalog catalog, @Injectable PaloAuth auth) // host: 127.0.0.1, port: 7777, without driver and odbc_type CreateResourceStmt stmt = new CreateResourceStmt(true, name, properties); stmt.analyze(analyzer); - ExternalCatalogResource resource = (ExternalCatalogResource) Resource.fromStmt(stmt); + OdbcCatalogResource resource = (OdbcCatalogResource) Resource.fromStmt(stmt); Assert.assertEquals(name, resource.getName()); Assert.assertEquals(type, resource.getType().name().toLowerCase()); Assert.assertEquals(host, resource.getProperties("host")); @@ -92,7 +93,7 @@ public void testFromStmt(@Mocked Catalog catalog, @Injectable PaloAuth auth) properties.put("odbc_type", "mysql"); stmt = new CreateResourceStmt(true, name, properties); stmt.analyze(analyzer); - resource = (ExternalCatalogResource) Resource.fromStmt(stmt); + resource = (OdbcCatalogResource) Resource.fromStmt(stmt); Assert.assertEquals("mysql", resource.getProperties("driver")); Assert.assertEquals("mysql", resource.getProperties("odbc_type")); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 670bd2a33d7684..058da4b2dd1182 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -360,7 +360,7 @@ public static void beforeClass() throws Exception { "\"database\" = \"db1\",\n" + "\"table\" = \"tbl1\",\n" + "\"driver\" = \"Oracle Driver\",\n" + - "\"type\" = \"mysql\"\n" + + "\"odbc_type\" = \"mysql\"\n" + ");"); } From 653f8422e57d3dd4d2f646e69b98d1c36afe8a75 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Thu, 24 Sep 2020 16:40:00 +0800 Subject: [PATCH 3/3] add UT of ODBC catalog resource serialization --- .../catalog/OdbcCatalogResourceTest.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java index 1b48a2417ff273..9f21f7f1df9fa0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java @@ -21,10 +21,13 @@ import org.apache.doris.analysis.AccessTestUtil; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.CreateResourceStmt; +import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; import org.apache.doris.common.proc.BaseProcResult; +import org.apache.doris.meta.MetaContext; import org.apache.doris.mysql.privilege.PaloAuth; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.persist.DropInfo; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Maps; @@ -35,6 +38,12 @@ import org.junit.Before; import org.junit.Test; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.HashMap; import java.util.Map; public class OdbcCatalogResourceTest { @@ -102,4 +111,49 @@ public void testFromStmt(@Mocked Catalog catalog, @Injectable PaloAuth auth) resource.getProcNodeData(result); Assert.assertEquals(7, result.getRows().size()); } + + @Test + public void testSerialization() throws Exception { + MetaContext metaContext = new MetaContext(); + metaContext.setMetaVersion(FeMetaVersion.VERSION_92); + metaContext.setThreadLocalInfo(); + + // 1. Write objects to file + File file = new File("./odbcCatalogResource"); + file.createNewFile(); + DataOutputStream dos = new DataOutputStream(new FileOutputStream(file)); + + OdbcCatalogResource odbcCatalogResource1 = new OdbcCatalogResource("odbc1"); + odbcCatalogResource1.write(dos); + + Map configs = new HashMap<>(); + configs.put("host", "host"); + configs.put("port", "port"); + configs.put("user", "user"); + configs.put("password", "password"); + OdbcCatalogResource odbcCatalogResource2 = new OdbcCatalogResource("odbc2"); + odbcCatalogResource2.setProperties(configs); + odbcCatalogResource2.write(dos); + + dos.flush(); + dos.close(); + + // 2. Read objects from file + DataInputStream dis = new DataInputStream(new FileInputStream(file)); + + OdbcCatalogResource rOdbcCatalogResource1 = (OdbcCatalogResource) OdbcCatalogResource.read(dis); + OdbcCatalogResource rOdbcCatalogResource2 = (OdbcCatalogResource) OdbcCatalogResource.read(dis); + + Assert.assertEquals("odbc1", rOdbcCatalogResource1.getName()); + Assert.assertEquals("odbc2", rOdbcCatalogResource2.getName()); + + Assert.assertEquals(rOdbcCatalogResource2.getProperties("host"), "host"); + Assert.assertEquals(rOdbcCatalogResource2.getProperties("port"), "port"); + Assert.assertEquals(rOdbcCatalogResource2.getProperties("user"), "user"); + Assert.assertEquals(rOdbcCatalogResource2.getProperties("password"), "password"); + + // 3. delete files + dis.close(); + file.delete(); + } } \ No newline at end of file