From 5c47e5b5893b2bd15298c1f9cdc3266899654f0a Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 22 Aug 2024 00:22:57 +0800 Subject: [PATCH 1/2] 1 --- .../apache/doris/datasource/InternalCatalog.java | 5 +++-- .../jdbc/test_mysql_jdbc_catalog.out | 10 ++++++++++ .../jdbc/test_mysql_jdbc_catalog.groovy | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 6e3382aa8a155b..4d7cf9da40d3cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1371,8 +1371,9 @@ public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlExceptio if (resultExpr.getSrcSlotRef() != null && resultExpr.getSrcSlotRef().getTable() != null && !resultExpr.getSrcSlotRef().getTable().isManagedTable()) { - if (createTableStmt.getPartitionDesc().inIdentifierPartitions( - resultExpr.getSrcSlotRef().getColumnName()) + if ((createTableStmt.getPartitionDesc() != null + && createTableStmt.getPartitionDesc().inIdentifierPartitions( + resultExpr.getSrcSlotRef().getColumnName())) || (createTableStmt.getDistributionDesc() != null && createTableStmt.getDistributionDesc().inDistributionColumns( resultExpr.getSrcSlotRef().getColumnName()))) { diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out index c9448cb3bca3f2..869e885280cf7d 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out @@ -468,3 +468,13 @@ doris -- !sql -- +-- !sql -- +int_u bigint Yes true \N +text varchar(65533) Yes true \N +t2 text Yes false \N NONE + +-- !sql -- +int_u bigint Yes true \N +text varchar(65533) Yes true \N +t2 varchar(65533) Yes false \N NONE + diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy index 888edcc0fcd322..2dd7b6b09b492f 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy @@ -612,6 +612,20 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc order_qt_sql """select * from mysql_conjuncts.doris_test.text_push where pk <=7;""" + // test create table as select + sql """use internal.${internal_db_name}""" + sql """drop table if exists ctas_partition_text_1""" + sql """drop table if exists ctas_partition_text_2""" + sql """set enable_nereids_planner=true""" + sql """create table ctas_partition_text_1 distributed by hash(text) buckets 1 properties("replication_num" = "1") as select int_u, text, text as t2 from mysql_conjuncts.doris_test.all_types;""" + qt_sql """desc ctas_partition_text_1""" + // ctas logic is different between new and old planner. + // so need to test both. + // the old planner's test can be removed once the old planner is removed. + sql """set enable_nereids_planner=false""" + sql """create table ctas_partition_text_2 distributed by hash(text) buckets 1 properties("replication_num" = "1") as select int_u, text, text as t2 from mysql_conjuncts.doris_test.all_types;""" + qt_sql """desc ctas_partition_text_2""" + sql """drop catalog if exists mysql_conjuncts;""" } } From 5f89a50bde2977186658a5b90db48099246fd00f Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 22 Aug 2024 18:23:33 +0800 Subject: [PATCH 2/2] fix --- .../plans/commands/CreateTableCommand.java | 17 ++++++++++++++--- .../jdbc/test_mysql_jdbc_catalog.out | 8 ++++++++ .../jdbc/test_mysql_jdbc_catalog.groovy | 15 +++++++++++++-- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java index 3a1f0caa9e0adc..888399bcb6d25a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java @@ -121,6 +121,8 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { Slot s = slots.get(i); DataType dataType = s.getDataType().conversion(); if (i == 0 && dataType.isStringType()) { + // first column of olap table can not be string type. + // So change it to varchar type. dataType = VarcharType.createVarcharType(ScalarType.MAX_VARCHAR_LENGTH); } else { dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, @@ -133,13 +135,21 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { if (createTableInfo.getPartitionTableInfo().inIdentifierPartitions(s.getName()) || (createTableInfo.getDistribution() != null && createTableInfo.getDistribution().inDistributionColumns(s.getName()))) { - // String type can not be used in partition/distributed column + // String type can not be used in partition/distributed column, // so we replace it to varchar dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, CharacterType.class, VarcharType.MAX_VARCHAR_TYPE); } else { - dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, - CharacterType.class, StringType.INSTANCE); + if (i == 0) { + // first column of olap table can not be string type. + // So change it to varchar type. + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + CharacterType.class, VarcharType.MAX_VARCHAR_TYPE); + } else { + // change varchar/char column from external table to string type + dataType = TypeCoercionUtils.replaceSpecifiedType(dataType, + CharacterType.class, StringType.INSTANCE); + } } } } else { @@ -215,3 +225,4 @@ public StmtType stmtType() { return StmtType.CREATE; } } + diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out index 869e885280cf7d..2be6d4a141f7fd 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out @@ -473,8 +473,16 @@ int_u bigint Yes true \N text varchar(65533) Yes true \N t2 text Yes false \N NONE +-- !sql -- +varchar varchar(65533) Yes true \N +int_u bigint Yes false \N NONE + -- !sql -- int_u bigint Yes true \N text varchar(65533) Yes true \N t2 varchar(65533) Yes false \N NONE +-- !sql -- +varchar varchar(65533) Yes true \N +int_u bigint Yes false \N NONE + diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy index 2dd7b6b09b492f..d30d7fe9150eff 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy @@ -23,6 +23,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc String s3_endpoint = getS3Endpoint() String bucket = getS3BucketName() String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar" + // String driver_url = "mysql-connector-java-8.0.25.jar" if (enabled != null && enabled.equalsIgnoreCase("true")) { String user = "test_jdbc_user"; String pwd = '123456'; @@ -616,15 +617,25 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc sql """use internal.${internal_db_name}""" sql """drop table if exists ctas_partition_text_1""" sql """drop table if exists ctas_partition_text_2""" + sql """drop table if exists ctas_partition_text_3""" + sql """drop table if exists ctas_partition_text_4""" sql """set enable_nereids_planner=true""" + // 1. test text type column as distribution col sql """create table ctas_partition_text_1 distributed by hash(text) buckets 1 properties("replication_num" = "1") as select int_u, text, text as t2 from mysql_conjuncts.doris_test.all_types;""" qt_sql """desc ctas_partition_text_1""" + // 2. test varchar type column as first col + sql """create table ctas_partition_text_2 distributed by hash(int_u) buckets 1 properties("replication_num" = "1") as select varchar, int_u from mysql_conjuncts.doris_test.all_types;""" + qt_sql """desc ctas_partition_text_2""" // ctas logic is different between new and old planner. // so need to test both. // the old planner's test can be removed once the old planner is removed. sql """set enable_nereids_planner=false""" - sql """create table ctas_partition_text_2 distributed by hash(text) buckets 1 properties("replication_num" = "1") as select int_u, text, text as t2 from mysql_conjuncts.doris_test.all_types;""" - qt_sql """desc ctas_partition_text_2""" + // 1. test text type column as distribution col + sql """create table ctas_partition_text_3 distributed by hash(text) buckets 1 properties("replication_num" = "1") as select int_u, text, text as t2 from mysql_conjuncts.doris_test.all_types;""" + qt_sql """desc ctas_partition_text_3""" + // 2. test varchar type column as first col + sql """create table ctas_partition_text_4 distributed by hash(int_u) buckets 1 properties("replication_num" = "1") as select varchar, int_u from mysql_conjuncts.doris_test.all_types;""" + qt_sql """desc ctas_partition_text_4""" sql """drop catalog if exists mysql_conjuncts;""" }