Skip to content

[improvement](ctas) use string type for varchar/char/string #14382

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 3 commits into from
Nov 18, 2022
Merged

[improvement](ctas) use string type for varchar/char/string #14382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a0a5458
[improvement](ctas) use string type for varchar/char/string
morningman Nov 17, 2022
5a0e170
add doc
morningman Nov 17, 2022
d96d570
fix test
morningman Nov 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...anual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE-AS-SELECT.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ illustrate:

<version since='1.2'>

- All columns of type string (varchar/var/string) are created as type "string".
- If the created source is an external table and the first column is of type String, the first column is automatically set to VARCHAR(65533). Because of Doris internal table, String column is not allowed as first column.

</version>
Expand Down
1 change: 1 addition & 0 deletions ...anual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE-AS-SELECT.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ CREATE TABLE table_name [( column_name_list )]

<version since='1.2'>

- 所有字符串类型的列(varchar/var/string) 都会被创建为 string 类型。
- 如果创建的来源为外部表，并且第一列为 String 类型，则会自动将第一列设置为 VARCHAR(65533)。因为 Doris 内部表，不允许 String 列作为第一列。

</version>
Expand Down
4 changes: 3 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1211,7 +1211,9 @@ public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlExceptio
TypeDef typeDef;
Expr resultExpr = resultExprs.get(i);
Type resultType = resultExpr.getType();
if (resultType.isStringType() && resultType.getLength() < 0) {
if (resultType.isStringType()) {
// Use String for varchar/char/string type,
// to avoid char-length-vs-byte-length issue.
typeDef = new TypeDef(ScalarType.createStringType());
} else if (resultType.isDecimalV2() && resultType.equals(ScalarType.DECIMALV2)) {
typeDef = new TypeDef(ScalarType.createDecimalType(27, 9));
Expand Down
48 changes: 25 additions & 23 deletions fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public void testDecimal() throws Exception {
String selectFromDecimal = "create table `test`.`select_decimal_table` PROPERTIES(\"replication_num\" = \"1\") "
+ "as select * from `test`.`decimal_table`";
createTableAsSelect(selectFromDecimal);
Assertions.assertEquals("CREATE TABLE `select_decimal_table` (\n" + " `userId` varchar(255) NOT NULL,\n"
Assertions.assertEquals("CREATE TABLE `select_decimal_table` (\n" + " `userId` varchar(65533) NOT NULL,\n"
+ " `amount_decimal` decimal(10, 2) NOT NULL\n"
+ ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n"
Expand Down Expand Up @@ -130,8 +130,8 @@ public void testVarchar() throws Exception {
+ "as select * from `test`.`varchar_table`";
createTableAsSelect(selectFromVarchar);
ShowResultSet showResultSet = showCreateTableByName("select_varchar");
Assertions.assertEquals("CREATE TABLE `select_varchar` (\n" + " `userId` varchar(255) NOT NULL,\n"
+ " `username` varchar(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n"
Assertions.assertEquals("CREATE TABLE `select_varchar` (\n" + " `userId` varchar(65533) NOT NULL,\n"
+ " `username` text NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
Expand Down Expand Up @@ -183,8 +183,8 @@ public void testAlias() throws Exception {
+ "as select userId as alias_name, username from `test`.`varchar_table`";
createTableAsSelect(selectAlias2);
ShowResultSet showResultSet2 = showCreateTableByName("select_alias_2");
Assertions.assertEquals("CREATE TABLE `select_alias_2` (\n" + " `alias_name` varchar(255) NOT NULL,\n"
+ " `username` varchar(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`alias_name`)\n"
Assertions.assertEquals("CREATE TABLE `select_alias_2` (\n" + " `alias_name` varchar(65533) NOT NULL,\n"
+ " `username` text NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`alias_name`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`alias_name`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");",
Expand All @@ -198,8 +198,8 @@ public void testJoin() throws Exception {
+ "join `test`.`join_table` jt on vt.userId=jt.userId";
createTableAsSelect(selectFromJoin);
ShowResultSet showResultSet = showCreateTableByName("select_join");
Assertions.assertEquals("CREATE TABLE `select_join` (\n" + " `userId` varchar(255) NOT NULL,\n"
+ " `username` varchar(255) NOT NULL,\n"
Assertions.assertEquals("CREATE TABLE `select_join` (\n" + " `userId` varchar(65533) NOT NULL,\n"
+ " `username` text NOT NULL,\n"
+ " `status` int(11) NOT NULL\n" + ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
Expand All @@ -212,8 +212,8 @@ public void testJoin() throws Exception {
+ "from `test`.`varchar_table` vt " + "join `test`.`join_table` jt on vt.userId=jt.userId";
createTableAsSelect(selectFromJoin1);
ShowResultSet showResultSet1 = showCreateTableByName("select_join1");
Assertions.assertEquals("CREATE TABLE `select_join1` (\n" + " `userId1` varchar(255) NOT NULL,\n"
+ " `userId2` varchar(255) NOT NULL,\n" + " `username` varchar(255) NOT NULL,\n"
Assertions.assertEquals("CREATE TABLE `select_join1` (\n" + " `userId1` varchar(65533) NOT NULL,\n"
+ " `userId2` text NOT NULL,\n" + " `username` text NOT NULL,\n"
+ " `status` int(11) NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId1`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId1`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
Expand All @@ -229,8 +229,8 @@ public void testName() throws Exception {
+ "join `test`.`join_table` jt on vt.userId=jt.userId";
createTableAsSelect(selectFromName);
ShowResultSet showResultSet = showCreateTableByName("select_name");
Assertions.assertEquals("CREATE TABLE `select_name` (\n" + " `user` varchar(255) NOT NULL,\n"
+ " `testname` varchar(255) NOT NULL,\n"
Assertions.assertEquals("CREATE TABLE `select_name` (\n" + " `user` varchar(65533) NOT NULL,\n"
+ " `testname` text NOT NULL,\n"
+ " `userstatus` int(11) NOT NULL\n" + ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`user`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`user`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n"
Expand All @@ -245,11 +245,13 @@ public void testUnion() throws Exception {
+ "as select userId from `test`.`varchar_table` union select userId from `test`.`join_table`";
createTableAsSelect(selectFromName);
ShowResultSet showResultSet = showCreateTableByName("select_union");
Assertions.assertEquals("CREATE TABLE `select_union` (\n" + " `userId` varchar(255) NULL\n" + ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\""
+ " = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1));
Assertions.assertEquals(
"CREATE TABLE `select_union` (\n" + " `userId` varchar(65533) NULL\n" + ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n"
+ "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n"
+ "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\""
+ " = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1));
}

@Test
Expand All @@ -259,7 +261,7 @@ public void testCte() throws Exception {
createTableAsSelect(selectFromCte);
ShowResultSet showResultSet = showCreateTableByName("select_cte");
Assertions.assertEquals(
"CREATE TABLE `select_cte` (\n" + " `userId` varchar(255) NOT NULL\n" + ") ENGINE=OLAP\n"
"CREATE TABLE `select_cte` (\n" + " `userId` varchar(65533) NOT NULL\n" + ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
Expand All @@ -283,8 +285,8 @@ public void testPartition() throws Exception {
+ "PROPERTIES (\"replication_num\" = \"1\") as " + "select * from `test`.`varchar_table`;";
createTableAsSelect(selectFromPartition);
ShowResultSet showResultSet = showCreateTableByName("selectPartition");
Assertions.assertEquals("CREATE TABLE `selectPartition` (\n" + " `userId` varchar(255) NOT NULL,\n"
+ " `username` varchar(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n"
Assertions.assertEquals("CREATE TABLE `selectPartition` (\n" + " `userId` varchar(65533) NOT NULL,\n"
+ " `username` text NOT NULL\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n"
+ "COMMENT 'OLAP'\n" + "PARTITION BY LIST(`userId`)\n"
+ "(PARTITION p1 VALUES IN (\"CA\",\"GB\",\"US\",\"ZH\"))\n"
+ "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n"
Expand All @@ -299,7 +301,7 @@ public void testDefaultTimestamp() throws Exception {
+ " as select * from `test`.`default_timestamp_table`";
createTableAsSelect(createSql);
ShowResultSet showResultSet = showCreateTableByName("test_default_timestamp");
Assertions.assertEquals("CREATE TABLE `test_default_timestamp` (\n" + " `userId` varchar(255) NOT NULL,\n"
Assertions.assertEquals("CREATE TABLE `test_default_timestamp` (\n" + " `userId` varchar(65533) NOT NULL,\n"
+ " `date` datetime NULL DEFAULT CURRENT_TIMESTAMP\n"
+ ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n"
Expand All @@ -315,7 +317,7 @@ public void testAggValue() throws Exception {
createTableAsSelect(createSql);
ShowResultSet showResultSet = showCreateTableByName("test_agg_value");
Assertions.assertEquals(
"CREATE TABLE `test_agg_value` (\n" + " `username` varchar(255) NOT NULL\n" + ") ENGINE=OLAP\n"
"CREATE TABLE `test_agg_value` (\n" + " `username` varchar(65533) NOT NULL\n" + ") ENGINE=OLAP\n"
+ "DUPLICATE KEY(`username`)\n" + "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`username`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
Expand All @@ -330,8 +332,8 @@ public void testUseKeyType() throws Exception {
createTableAsSelect(createSql);
ShowResultSet showResultSet = showCreateTableByName("test_use_key_type");
Assertions.assertEquals(
"CREATE TABLE `test_use_key_type` (\n" + " `userId` varchar(255) NOT NULL,\n"
+ " `username` varchar(255) NOT NULL\n" + ") ENGINE=OLAP\n" + "UNIQUE KEY(`userId`)\n"
"CREATE TABLE `test_use_key_type` (\n" + " `userId` varchar(65533) NOT NULL,\n"
+ " `username` text NOT NULL\n" + ") ENGINE=OLAP\n" + "UNIQUE KEY(`userId`)\n"
+ "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n"
+ "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n"
+ "\"storage_format\" = \"V2\",\n" + "\"disable_auto_compaction\" = \"false\"\n" + ");",
Expand Down
4 changes: 2 additions & 2 deletions regression-test/data/ddl_p0/test_ctas.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select --
test_ctas1 CREATE TABLE `test_ctas1` (\n `test_varchar` varchar(150) NULL,\n `test_text` text NULL,\n `test_datetime` datetime NULL,\n `test_default_timestamp` datetime NULL DEFAULT CURRENT_TIMESTAMP\n) ENGINE=OLAP\nDUPLICATE KEY(`test_varchar`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`test_varchar`) BUCKETS 10\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n);
test_ctas1 CREATE TABLE `test_ctas1` (\n `test_varchar` varchar(65533) NULL,\n `test_text` text NULL,\n `test_datetime` datetime NULL,\n `test_default_timestamp` datetime NULL DEFAULT CURRENT_TIMESTAMP\n) ENGINE=OLAP\nDUPLICATE KEY(`test_varchar`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`test_varchar`) BUCKETS 10\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n);

-- !select --
2

-- !select --
test_ctas2 CREATE TABLE `test_ctas2` (\n `test_varchar` varchar(150) NULL,\n `test_text` text NULL,\n `test_datetime` datetime NULL,\n `test_default_timestamp` datetime NULL DEFAULT CURRENT_TIMESTAMP\n) ENGINE=OLAP\nDUPLICATE KEY(`test_varchar`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`test_varchar`) BUCKETS 10\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n);
test_ctas2 CREATE TABLE `test_ctas2` (\n `test_varchar` varchar(65533) NULL,\n `test_text` text NULL,\n `test_datetime` datetime NULL,\n `test_default_timestamp` datetime NULL DEFAULT CURRENT_TIMESTAMP\n) ENGINE=OLAP\nDUPLICATE KEY(`test_varchar`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`test_varchar`) BUCKETS 10\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n);

-- !select --
2
Expand Down