From df347126f2a2ef8169953ea2aad6b9d39b10efa9 Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 27 Jun 2021 17:05:09 +0800 Subject: [PATCH 01/10] =?UTF-8?q?ADD:=20=E6=B7=BB=E5=8A=A0create=20table?= =?UTF-8?q?=20as=20select?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fe/fe-core/src/main/cup/sql_parser.cup | 15 ++ .../analysis/CreateTableAsSelectStmt.java | 79 ++---- .../doris/analysis/CreateTableStmt.java | 4 + .../org/apache/doris/analysis/SelectStmt.java | 2 +- .../org/apache/doris/catalog/Catalog.java | 69 +++++ .../java/org/apache/doris/qe/DdlExecutor.java | 3 + .../org/apache/doris/qe/StmtExecutor.java | 25 +- .../analysis/CreateTableAsSelectStmtTest.java | 246 ++++++++++++++++++ 8 files changed, 366 insertions(+), 77 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index f4e573d4253bc1..34bb55484e49e7 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -1236,6 +1236,21 @@ create_stmt ::= RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, indexes, engineName, keys, partition, distribution, tblProperties, extProperties, tableComment, index); :} + | KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name + opt_col_list:columns + opt_engine:engineName + opt_keys:keys + opt_comment:tableComment + opt_partition:partition + opt_distribution:distribution + opt_rollup:index + opt_properties:tblProperties + opt_ext_properties:extProperties + KW_AS select_stmt:table_def + {: + RESULT = new CreateTableAsSelectStmt(new CreateTableStmt(ifNotExists, isExternal, name, null, engineName, keys, partition, + distribution, tblProperties, extProperties, tableComment, index), columns, table_def); + :} /* User */ | KW_CREATE KW_USER opt_if_not_exists:ifNotExists grant_user:user opt_user_role:userRole {: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java index 9f2c188ec4e2d9..c4005dab6464d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java @@ -17,8 +17,6 @@ package org.apache.doris.analysis; -import org.apache.doris.catalog.Table; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; @@ -31,78 +29,45 @@ * CREATE TABLE table_name [( column_name_list )] * opt_engine opt_partition opt_properties KW_AS query_stmt */ -public class CreateTableAsSelectStmt extends StatementBase { +public class CreateTableAsSelectStmt extends DdlStmt { private final CreateTableStmt createTableStmt; private final List columnNames; - private final QueryStmt queryStmt; - private final InsertStmt insertStmt; - + private SelectStmt selectStmt; + public CreateTableAsSelectStmt(CreateTableStmt createTableStmt, - List columnNames, - QueryStmt queryStmt) { + List columnNames, SelectStmt selectStmt) { this.createTableStmt = createTableStmt; this.columnNames = columnNames; - this.queryStmt = queryStmt; - this.insertStmt = new InsertStmt(createTableStmt.getDbTbl(), queryStmt); + this.selectStmt = selectStmt; + // Insert is not currently supported } - + @Override - public void analyze(Analyzer analyzer) throws UserException, AnalysisException { + public void analyze(Analyzer analyzer) throws UserException { // first: we analyze queryStmt before create table. // To avoid duplicate registrations of table/colRefs, // create a new root analyzer and clone the query statement for this initial pass. Analyzer dummyRootAnalyzer = new Analyzer(analyzer.getCatalog(), analyzer.getContext()); - QueryStmt tmpStmt = queryStmt.clone(); + SelectStmt tmpStmt = selectStmt.clone(); tmpStmt.analyze(dummyRootAnalyzer); - - // TODO(zc): support char, varchar and decimal + this.selectStmt = tmpStmt; + // TODO: support decimal for (Expr expr : tmpStmt.getResultExprs()) { - if (expr.getType().isDecimalV2() || expr.getType().isStringType()) { + if (expr.getType().isDecimalV2()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_UNSUPPORTED_TYPE_IN_CTAS, expr.getType()); } } - - // Check columnNames - if (columnNames != null) { - if (columnNames.size() != tmpStmt.getColLabels().size()) { - ErrorReport.report(ErrorCode.ERR_COL_NUMBER_NOT_MATCH); - } - for (int i = 0; i < columnNames.size(); ++i) { - createTableStmt.addColumnDef(new ColumnDef( - columnNames.get(i), new TypeDef(tmpStmt.getResultExprs().get(i).getType()))); - } - } else { - for (int i = 0; i < tmpStmt.getColLabels().size(); ++i) { - createTableStmt.addColumnDef(new ColumnDef( - tmpStmt.getColLabels().get(i), new TypeDef(tmpStmt.getResultExprs().get(i).getType()))); - } - } - - // Analyze create table statement - createTableStmt.analyze(analyzer); - - // Analyze insert - Table newTable = null; - insertStmt.setTargetTable(newTable); - insertStmt.analyze(analyzer); } - - public void createTable(Analyzer analyzer) throws AnalysisException { - // TODO(zc): Support create table later. - // Create table - try { - analyzer.getCatalog().createTable(createTableStmt); - } catch (UserException e) { - throw new AnalysisException(e.getMessage()); - } + + public CreateTableStmt getCreateTableStmt() { + return createTableStmt; } - - public InsertStmt getInsertStmt() { - return insertStmt; + + public List getColumnNames() { + return columnNames; } - - @Override - public RedirectStatus getRedirectStatus() { - return RedirectStatus.FORWARD_WITH_SYNC; + + public SelectStmt getSelectStmt() { + return selectStmt; } -} +} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 78fb1550e39015..854b81abe26d8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -203,6 +203,10 @@ public DistributionDesc getDistributionDesc() { return this.distributionDesc; } + public void setDistributionDesc(DistributionDesc desc) { + this.distributionDesc = desc; + } + public Map getProperties() { return this.properties; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index 0f2167e789ad13..0f10a287938eee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -188,7 +188,7 @@ public void reset() { } @Override - public QueryStmt clone() { + public SelectStmt clone() { return new SelectStmt(this); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index bd8bf344da4304..d2e461c394f2c9 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -40,15 +40,18 @@ import org.apache.doris.analysis.AlterSystemStmt; import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.AlterViewStmt; +import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.BackupStmt; import org.apache.doris.analysis.CancelAlterSystemStmt; import org.apache.doris.analysis.CancelAlterTableStmt; import org.apache.doris.analysis.CancelBackupStmt; +import org.apache.doris.analysis.ColumnDef; import org.apache.doris.analysis.ColumnRenameClause; import org.apache.doris.analysis.CreateClusterStmt; import org.apache.doris.analysis.CreateDbStmt; import org.apache.doris.analysis.CreateFunctionStmt; import org.apache.doris.analysis.CreateMaterializedViewStmt; +import org.apache.doris.analysis.CreateTableAsSelectStmt; import org.apache.doris.analysis.CreateTableLikeStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.CreateUserStmt; @@ -62,7 +65,10 @@ import org.apache.doris.analysis.DropMaterializedViewStmt; import org.apache.doris.analysis.DropPartitionClause; import org.apache.doris.analysis.DropTableStmt; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.FunctionName; +import org.apache.doris.analysis.HashDistributionDesc; import org.apache.doris.analysis.InstallPluginStmt; import org.apache.doris.analysis.KeysDesc; import org.apache.doris.analysis.LinkDbStmt; @@ -76,12 +82,15 @@ import org.apache.doris.analysis.ReplacePartitionClause; import org.apache.doris.analysis.RestoreStmt; import org.apache.doris.analysis.RollupRenameClause; +import org.apache.doris.analysis.SelectListItem; +import org.apache.doris.analysis.SelectStmt; import org.apache.doris.analysis.ShowAlterStmt.AlterType; import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.TableRenameClause; import org.apache.doris.analysis.TruncateTableStmt; +import org.apache.doris.analysis.TypeDef; import org.apache.doris.analysis.UninstallPluginStmt; import org.apache.doris.analysis.UserDesc; import org.apache.doris.analysis.UserIdentity; @@ -249,6 +258,7 @@ import com.sleepycat.je.rep.NetworkRestoreConfig; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; @@ -3086,6 +3096,65 @@ public void createTableLike(CreateTableLikeStmt stmt) throws DdlException { throw new DdlException("Failed to execute CREATE TABLE LIKE " + stmt.getExistedTableName() + ". Reason: " + e.getMessage()); } } + + public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlException { + try { + SelectStmt selectStmt = stmt.getSelectStmt(); + List columnNames = stmt.getColumnNames(); + CreateTableStmt createTableStmt = stmt.getCreateTableStmt(); + List items = selectStmt.getSelectList().getItems(); + ArrayList resultExprs = selectStmt.getResultExprs(); + int size = items.size(); + // Check columnNames + if (columnNames != null && columnNames.size() != size) { + ErrorReport.report(ErrorCode.ERR_COL_NUMBER_NOT_MATCH); + } else { + int colNameIndex = 0; + for (int i = 0; i < size; ++i) { + String name; + if (columnNames != null) { + // use custom column names + name = columnNames.get(i); + } else { + SelectListItem selectListItem = items.get(i); + Expr expr = selectListItem.getExpr(); + if (expr instanceof FunctionCallExpr) { + // function use alias or generate name + String alias = selectListItem.getAlias(); + if (StringUtils.isNotEmpty(alias)) { + name = alias; + } else { + name = "_col" + (colNameIndex++); + } + } else { + name = selectStmt.getColLabels().get(i); + } + } + TypeDef typeDef; + Expr resultExpr = resultExprs.get(i); + // varchar/char transfer to string + if (resultExpr.getType().isStringType()) { + typeDef = new TypeDef(Type.STRING); + } else { + typeDef = new TypeDef(resultExpr.getType()); + } + createTableStmt.addColumnDef(new ColumnDef(name, typeDef, false, + null, true, + new ColumnDef.DefaultValue(false, null), + "")); + // set first column as default distribution + if (createTableStmt.getDistributionDesc() == null && i == 0) { + createTableStmt.setDistributionDesc(new HashDistributionDesc(10, Lists.newArrayList(name))); + } + } + } + Analyzer dummyRootAnalyzer = new Analyzer(this, ConnectContext.get()); + createTableStmt.analyze(dummyRootAnalyzer); + createTable(createTableStmt); + } catch (UserException e) { + throw new DdlException("Failed to execute CREATE TABLE AS SELECT Reason: " + e.getMessage()); + } + } public void addPartition(Database db, String tableName, AddPartitionClause addPartitionClause) throws DdlException { SinglePartitionDesc singlePartitionDesc = addPartitionClause.getSingeRangePartitionDesc(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index 89536ee5fdbc7c..d4f75ba9dde6b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -51,6 +51,7 @@ import org.apache.doris.analysis.CreateRoleStmt; import org.apache.doris.analysis.CreateRoutineLoadStmt; import org.apache.doris.analysis.CreateSqlBlockRuleStmt; +import org.apache.doris.analysis.CreateTableAsSelectStmt; import org.apache.doris.analysis.CreateTableLikeStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.CreateUserStmt; @@ -126,6 +127,8 @@ public static void execute(Catalog catalog, DdlStmt ddlStmt) throws Exception { catalog.createTable((CreateTableStmt) ddlStmt); } else if (ddlStmt instanceof CreateTableLikeStmt) { catalog.createTableLike((CreateTableLikeStmt) ddlStmt); + } else if (ddlStmt instanceof CreateTableAsSelectStmt) { + catalog.createTableAsSelect((CreateTableAsSelectStmt) ddlStmt); } else if (ddlStmt instanceof DropTableStmt) { catalog.dropTable((DropTableStmt) ddlStmt); } else if (ddlStmt instanceof CreateMaterializedViewStmt) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index ec3a4c7bebfa67..72ec864cb57e46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -367,8 +367,6 @@ public void execute(TUniqueId queryId) throws Exception { handleUseStmt(); } else if (parsedStmt instanceof TransactionStmt) { handleTransactionStmt(); - } else if (parsedStmt instanceof CreateTableAsSelectStmt) { - handleInsertStmt(); } else if (parsedStmt instanceof InsertStmt) { // Must ahead of DdlStmt because InserStmt is its subclass try { handleInsertStmt(); @@ -511,13 +509,12 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException { if (parsedStmt instanceof QueryStmt) { queryStmt = (QueryStmt) parsedStmt; queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + } else if (parsedStmt instanceof CreateTableAsSelectStmt) { + CreateTableAsSelectStmt parsedStmt = (CreateTableAsSelectStmt) this.parsedStmt; + queryStmt = parsedStmt.getSelectStmt(); + queryStmt.getTables(analyzer, tableMap, parentViewNameSet); } else { - InsertStmt insertStmt; - if (parsedStmt instanceof InsertStmt) { - insertStmt = (InsertStmt) parsedStmt; - } else { - insertStmt = ((CreateTableAsSelectStmt) parsedStmt).getInsertStmt(); - } + InsertStmt insertStmt = (InsertStmt) parsedStmt; insertStmt.getTables(analyzer, tableMap, parentViewNameSet); } // table id in tableList is in ascending order because that table map is a sorted map @@ -641,9 +638,6 @@ private void analyzeAndGenerateQueryPlan(TQueryOptions tQueryOptions) throws Use planner = new Planner(); if (parsedStmt instanceof QueryStmt || parsedStmt instanceof InsertStmt) { planner.plan(parsedStmt, analyzer, tQueryOptions); - } else { - planner.plan(((CreateTableAsSelectStmt) parsedStmt).getInsertStmt(), - analyzer, new TQueryOptions()); } // TODO(zc): // Preconditions.checkState(!analyzer.hasUnassignedConjuncts()); @@ -1155,14 +1149,7 @@ private void handleInsertStmt() throws Exception { // Every time set no send flag and clean all data in buffer context.getMysqlChannel().reset(); // create plan - InsertStmt insertStmt = null; - if (parsedStmt instanceof CreateTableAsSelectStmt) { - // Create table here - ((CreateTableAsSelectStmt) parsedStmt).createTable(analyzer); - insertStmt = ((CreateTableAsSelectStmt) parsedStmt).getInsertStmt(); - } else { - insertStmt = (InsertStmt) parsedStmt; - } + InsertStmt insertStmt = (InsertStmt) parsedStmt; if (insertStmt.getQueryStmt().hasOutFileClause()) { throw new DdlException("Not support OUTFILE clause in INSERT statement"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java new file mode 100644 index 00000000000000..ab2448a4f1cfe2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -0,0 +1,246 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Catalog; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.metric.MetricRepo; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowExecutor; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.utframe.UtFrameUtils; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.UUID; + +public class CreateTableAsSelectStmtTest { + + private static String runningDir = "fe/mocked/CreateTableAsSelectStmtTest/" + UUID.randomUUID() + "/"; + private static ConnectContext connectContext; + + @AfterClass + public static void tearDown() throws Exception { + UtFrameUtils.cleanDorisFeDir(runningDir); + } + + @BeforeClass + public static void setUp() throws Exception { + UtFrameUtils.createMinDorisCluster(runningDir); + // create connect context + connectContext = UtFrameUtils.createDefaultCtx(); + // create database + String createDbStmtStr = "create database test"; + CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext); + Catalog.getCurrentCatalog().createDb(createDbStmt); + String varcharTable = "CREATE TABLE `test`.`varchar_table`\n" + + "(\n" + + " `userId` varchar(255) NOT NULL COMMENT '',\n" + + " `username` varchar(255) NOT NULL COMMENT ''\n" + + ") ENGINE = OLAP unique KEY(`userId`)\n" + + "COMMENT \"varchar_table\"\n" + + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");"; + String decimalTable = "CREATE TABLE `test`.`decimal_table`\n" + + "(\n" + + " `userId` varchar(255) NOT NULL COMMENT '',\n" + + " `amount_decimal` decimal(10, 2) NOT NULL COMMENT ''\n" + + ") ENGINE = OLAP unique KEY(`userId`)\n" + + "COMMENT \"decimal_table\"\n" + + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");"; + String joinTable = "CREATE TABLE `test`.`join_table`\n" + + "(\n" + + " `userId` varchar(255) NOT NULL COMMENT '',\n" + + " `status` int NOT NULL COMMENT ''\n" + + ") ENGINE = OLAP unique KEY(`userId`)\n" + + "COMMENT \"join_table\"\n" + + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");"; + MetricRepo.init(); + createTable(varcharTable); + createTable(decimalTable); + createTable(joinTable); + } + + private static void createTable(String sql) throws Exception { + CreateTableStmt stmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); + Catalog.getCurrentCatalog().createTable(stmt); + } + + private static void createTableAsSelect(String sql) throws Exception { + CreateTableAsSelectStmt stmt = (CreateTableAsSelectStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext); + Catalog.getCurrentCatalog().createTableAsSelect(stmt); + } + + private static ShowResultSet showCreateTable(String tableName) throws Exception { + ShowCreateTableStmt stmt = new ShowCreateTableStmt(new TableName("test", tableName)); + Analyzer dummyRootAnalyzer = new Analyzer(Catalog.getCurrentCatalog(), connectContext); + stmt.analyze(dummyRootAnalyzer); + ShowExecutor executor = new ShowExecutor(connectContext, stmt); + return executor.execute(); + } + + @Test + public void testErrorType() throws Exception { + String selectFromDecimal = "create table `test`.`select_decimal_table` PROPERTIES(\"replication_num\" = \"1\") as select * from `test`.`decimal_table`;\n"; + ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Unsupported type", + () -> UtFrameUtils.parseAndAnalyzeStmt(selectFromDecimal, connectContext)); + } + + @Test + public void testVarchar() throws Exception { + String selectFromDecimal = "create table `test`.`select_varchar` PROPERTIES(\"replication_num\" = \"1\") as select * from `test`.`varchar_table`;\n"; + createTableAsSelect(selectFromDecimal); + ShowResultSet showResultSet = showCreateTable("select_varchar"); + Assert.assertEquals("CREATE TABLE `select_varchar` (\n" + + " `userId` string NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`userId`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");", showResultSet.getResultRows().get(0).get(1)); + } + + @Test + public void testFunction() throws Exception { + String selectFromFunction1 = "create table `test`.`select_function_1` PROPERTIES(\"replication_num\" = \"1\") as select count(*) from `test`.`varchar_table`;\n"; + createTableAsSelect(selectFromFunction1); + ShowResultSet showResultSet1 = showCreateTable("select_function_1"); + Assert.assertEquals("CREATE TABLE `select_function_1` (\n" + + " `_col0` bigint(20) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`_col0`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");", showResultSet1.getResultRows().get(0).get(1)); + + String selectFromFunction2 = "create table `test`.`select_function_2` PROPERTIES(\"replication_num\" = \"1\") as select sum(status) from `test`.`join_table`;\n"; + createTableAsSelect(selectFromFunction2); + ShowResultSet showResultSet2 = showCreateTable("select_function_2"); + Assert.assertEquals("CREATE TABLE `select_function_2` (\n" + + " `_col0` bigint(20) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`_col0`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");", showResultSet2.getResultRows().get(0).get(1)); + } + + @Test + public void testAlias() throws Exception { + String selectAlias1 = "create table `test`.`select_alias_1` PROPERTIES(\"replication_num\" = \"1\") as select count(*) as amount from `test`.`varchar_table`;\n"; + createTableAsSelect(selectAlias1); + ShowResultSet showResultSet1 = showCreateTable("select_alias_1"); + Assert.assertEquals("CREATE TABLE `select_alias_1` (\n" + + " `amount` bigint(20) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`amount`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`amount`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");", showResultSet1.getResultRows().get(0).get(1)); + String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") as select userId as alias_name from `test`.`varchar_table`;\n"; + createTableAsSelect(selectAlias2); + ShowResultSet showResultSet2 = showCreateTable("select_alias_2"); + Assert.assertEquals("CREATE TABLE `select_alias_2` (\n" + + " `alias_name` string NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`alias_name`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`alias_name`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");", showResultSet2.getResultRows().get(0).get(1)); + } + + @Test + public void testJoin() throws Exception { + String selectFromJoin = "create table `test`.`select_join` PROPERTIES(\"replication_num\" = \"1\") " + + "as select vt.userId, vt.username, jt.status from `test`.`varchar_table` vt join `test`.`join_table` jt on vt.userId=jt.userId;\n"; + createTableAsSelect(selectFromJoin); + ShowResultSet showResultSet = showCreateTable("select_join"); + Assert.assertEquals("CREATE TABLE `select_join` (\n" + + " `userId` string NULL COMMENT \"\",\n" + + " `username` string NULL COMMENT \"\",\n" + + " `status` int(11) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`userId`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");", showResultSet.getResultRows().get(0).get(1)); + } + + @Test + public void testName() throws Exception { + String selectFromName = "create table `test`.`select_name`(user, testname, userstatus) PROPERTIES(\"replication_num\" = \"1\") " + + "as select vt.userId, vt.username, jt.status from `test`.`varchar_table` vt join `test`.`join_table` jt on vt.userId=jt.userId;\n"; + createTableAsSelect(selectFromName); + ShowResultSet showResultSet = showCreateTable("select_name"); + Assert.assertEquals("CREATE TABLE `select_name` (\n" + + " `user` string NULL COMMENT \"\",\n" + + " `testname` string NULL COMMENT \"\",\n" + + " `userstatus` int(11) NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`user`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`user`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");", showResultSet.getResultRows().get(0).get(1)); + } +} \ No newline at end of file From 72dc3d9afa036bf6a9dec66c77e2e82496c6f769 Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 28 Jun 2021 08:19:51 +0800 Subject: [PATCH 02/10] MOD: fix MOD --- .../org/apache/doris/analysis/CreateTableAsSelectStmt.java | 1 - fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java index c4005dab6464d5..452b091d5c6843 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java @@ -16,7 +16,6 @@ // under the License. package org.apache.doris.analysis; - import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index d2e461c394f2c9..688963857a82de 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -46,6 +46,7 @@ import org.apache.doris.analysis.CancelAlterTableStmt; import org.apache.doris.analysis.CancelBackupStmt; import org.apache.doris.analysis.ColumnDef; +import org.apache.doris.analysis.ColumnDef.DefaultValue; import org.apache.doris.analysis.ColumnRenameClause; import org.apache.doris.analysis.CreateClusterStmt; import org.apache.doris.analysis.CreateDbStmt; @@ -3140,7 +3141,7 @@ public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlExceptio } createTableStmt.addColumnDef(new ColumnDef(name, typeDef, false, null, true, - new ColumnDef.DefaultValue(false, null), + new DefaultValue(false, null), "")); // set first column as default distribution if (createTableStmt.getDistributionDesc() == null && i == 0) { From 4ef81ed8fb3b1ae2075a46ecc7815c993dde9811 Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 30 Aug 2021 12:59:22 +0800 Subject: [PATCH 03/10] MOD --- .../java/org/apache/doris/analysis/CreateTableAsSelectStmt.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java index 452b091d5c6843..c4005dab6464d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java @@ -16,6 +16,7 @@ // under the License. package org.apache.doris.analysis; + import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; From 752268a300340e08c31335c86341fcb63e83cfab Mon Sep 17 00:00:00 2001 From: stalary Date: Sun, 5 Sep 2021 11:38:38 +0800 Subject: [PATCH 04/10] MOD: finish code and test --- .../apache/doris/analysis/CreateTableAsSelectStmtTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index ab2448a4f1cfe2..21bfcf3f3bd74d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -45,7 +45,7 @@ public static void tearDown() throws Exception { @BeforeClass public static void setUp() throws Exception { - UtFrameUtils.createMinDorisCluster(runningDir); + UtFrameUtils.createDorisCluster(runningDir); // create connect context connectContext = UtFrameUtils.createDefaultCtx(); // create database @@ -243,4 +243,4 @@ public void testName() throws Exception { "\"storage_format\" = \"V2\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } -} \ No newline at end of file +} From af10dd89edb4fe0d12a6828d5e4dceeb93da3e64 Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 6 Sep 2021 10:58:56 +0800 Subject: [PATCH 05/10] MOD: mod some detail code --- .../java/org/apache/doris/analysis/CreateTableAsSelectStmt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java index c4005dab6464d5..dfc6fa4836c2b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java @@ -70,4 +70,4 @@ public List getColumnNames() { public SelectStmt getSelectStmt() { return selectStmt; } -} \ No newline at end of file +} From 8747a13afbd070052b2a452de41034569f8746ec Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 6 Sep 2021 11:43:08 +0800 Subject: [PATCH 06/10] MOD: unit test --- .../analysis/CreateTableAsSelectStmtTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 21bfcf3f3bd74d..7a752e6c0e4858 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -60,7 +60,7 @@ public static void setUp() throws Exception { "COMMENT \"varchar_table\"\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");"; @@ -72,7 +72,7 @@ public static void setUp() throws Exception { "COMMENT \"decimal_table\"\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");"; @@ -84,7 +84,7 @@ public static void setUp() throws Exception { "COMMENT \"join_table\"\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");"; @@ -131,7 +131,7 @@ public void testVarchar() throws Exception { "COMMENT \"OLAP\"\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); @@ -149,7 +149,7 @@ public void testFunction() throws Exception { "COMMENT \"OLAP\"\n" + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); @@ -164,7 +164,7 @@ public void testFunction() throws Exception { "COMMENT \"OLAP\"\n" + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); @@ -182,7 +182,7 @@ public void testAlias() throws Exception { "COMMENT \"OLAP\"\n" + "DISTRIBUTED BY HASH(`amount`) BUCKETS 10\n" + "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); @@ -196,7 +196,7 @@ public void testAlias() throws Exception { "COMMENT \"OLAP\"\n" + "DISTRIBUTED BY HASH(`alias_name`) BUCKETS 10\n" + "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); @@ -217,7 +217,7 @@ public void testJoin() throws Exception { "COMMENT \"OLAP\"\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); @@ -238,7 +238,7 @@ public void testName() throws Exception { "COMMENT \"OLAP\"\n" + "DISTRIBUTED BY HASH(`user`) BUCKETS 10\n" + "PROPERTIES (\n" + - "\"replication_num\" = \"1\",\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); From 3ea587f6e7269336f0cc8297797fe850a412d8fb Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 6 Sep 2021 22:38:35 +0800 Subject: [PATCH 07/10] MOD: name get way --- fe/fe-core/src/main/cup/sql_parser.cup | 4 +- .../analysis/CreateTableAsSelectStmt.java | 18 +++--- .../org/apache/doris/catalog/Catalog.java | 34 ++++------- .../org/apache/doris/qe/StmtExecutor.java | 2 +- .../analysis/CreateTableAsSelectStmtTest.java | 59 ++++++++++++++++--- 5 files changed, 78 insertions(+), 39 deletions(-) diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 34bb55484e49e7..351a890eaed123 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -1246,10 +1246,10 @@ create_stmt ::= opt_rollup:index opt_properties:tblProperties opt_ext_properties:extProperties - KW_AS select_stmt:table_def + KW_AS query_stmt:query_def {: RESULT = new CreateTableAsSelectStmt(new CreateTableStmt(ifNotExists, isExternal, name, null, engineName, keys, partition, - distribution, tblProperties, extProperties, tableComment, index), columns, table_def); + distribution, tblProperties, extProperties, tableComment, index), columns, query_def); :} /* User */ | KW_CREATE KW_USER opt_if_not_exists:ifNotExists grant_user:user opt_user_role:userRole diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java index dfc6fa4836c2b4..3623cf6780736c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java @@ -21,6 +21,7 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; +import java.util.ArrayList; import java.util.List; /** @@ -32,13 +33,13 @@ public class CreateTableAsSelectStmt extends DdlStmt { private final CreateTableStmt createTableStmt; private final List columnNames; - private SelectStmt selectStmt; + private QueryStmt queryStmt; public CreateTableAsSelectStmt(CreateTableStmt createTableStmt, - List columnNames, SelectStmt selectStmt) { + List columnNames, QueryStmt queryStmt) { this.createTableStmt = createTableStmt; this.columnNames = columnNames; - this.selectStmt = selectStmt; + this.queryStmt = queryStmt; // Insert is not currently supported } @@ -48,11 +49,12 @@ public void analyze(Analyzer analyzer) throws UserException { // To avoid duplicate registrations of table/colRefs, // create a new root analyzer and clone the query statement for this initial pass. Analyzer dummyRootAnalyzer = new Analyzer(analyzer.getCatalog(), analyzer.getContext()); - SelectStmt tmpStmt = selectStmt.clone(); + QueryStmt tmpStmt = queryStmt.clone(); tmpStmt.analyze(dummyRootAnalyzer); - this.selectStmt = tmpStmt; + this.queryStmt = tmpStmt; + ArrayList resultExprs = getQueryStmt().getResultExprs(); // TODO: support decimal - for (Expr expr : tmpStmt.getResultExprs()) { + for (Expr expr : resultExprs) { if (expr.getType().isDecimalV2()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_UNSUPPORTED_TYPE_IN_CTAS, expr.getType()); } @@ -67,7 +69,7 @@ public List getColumnNames() { return columnNames; } - public SelectStmt getSelectStmt() { - return selectStmt; + public QueryStmt getQueryStmt() { + return queryStmt; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 688963857a82de..1d9173230b8390 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -67,7 +67,6 @@ import org.apache.doris.analysis.DropPartitionClause; import org.apache.doris.analysis.DropTableStmt; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.FunctionName; import org.apache.doris.analysis.HashDistributionDesc; import org.apache.doris.analysis.InstallPluginStmt; @@ -77,16 +76,16 @@ import org.apache.doris.analysis.ModifyDistributionClause; import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.PartitionRenameClause; +import org.apache.doris.analysis.QueryStmt; import org.apache.doris.analysis.RecoverDbStmt; import org.apache.doris.analysis.RecoverPartitionStmt; import org.apache.doris.analysis.RecoverTableStmt; import org.apache.doris.analysis.ReplacePartitionClause; import org.apache.doris.analysis.RestoreStmt; import org.apache.doris.analysis.RollupRenameClause; -import org.apache.doris.analysis.SelectListItem; -import org.apache.doris.analysis.SelectStmt; import org.apache.doris.analysis.ShowAlterStmt.AlterType; import org.apache.doris.analysis.SinglePartitionDesc; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.TableRenameClause; @@ -123,6 +122,7 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.common.MetaHeader; import org.apache.doris.common.MetaNotFoundException; @@ -259,7 +259,6 @@ import com.sleepycat.je.rep.NetworkRestoreConfig; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; @@ -3100,12 +3099,12 @@ public void createTableLike(CreateTableLikeStmt stmt) throws DdlException { public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlException { try { - SelectStmt selectStmt = stmt.getSelectStmt(); List columnNames = stmt.getColumnNames(); CreateTableStmt createTableStmt = stmt.getCreateTableStmt(); - List items = selectStmt.getSelectList().getItems(); - ArrayList resultExprs = selectStmt.getResultExprs(); - int size = items.size(); + QueryStmt queryStmt = stmt.getQueryStmt(); + ArrayList resultExprs = queryStmt.getResultExprs(); + ArrayList colLabels = queryStmt.getColLabels(); + int size = resultExprs.size(); // Check columnNames if (columnNames != null && columnNames.size() != size) { ErrorReport.report(ErrorCode.ERR_COL_NUMBER_NOT_MATCH); @@ -3117,19 +3116,12 @@ public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlExceptio // use custom column names name = columnNames.get(i); } else { - SelectListItem selectListItem = items.get(i); - Expr expr = selectListItem.getExpr(); - if (expr instanceof FunctionCallExpr) { - // function use alias or generate name - String alias = selectListItem.getAlias(); - if (StringUtils.isNotEmpty(alias)) { - name = alias; - } else { - name = "_col" + (colNameIndex++); - } - } else { - name = selectStmt.getColLabels().get(i); - } + name = colLabels.get(i); + } + try { + FeNameFormat.checkColumnName(name); + } catch (AnalysisException exception) { + name = "_col" + (colNameIndex++); } TypeDef typeDef; Expr resultExpr = resultExprs.get(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 72ec864cb57e46..c7d32a4c9c5a71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -511,7 +511,7 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException { queryStmt.getTables(analyzer, tableMap, parentViewNameSet); } else if (parsedStmt instanceof CreateTableAsSelectStmt) { CreateTableAsSelectStmt parsedStmt = (CreateTableAsSelectStmt) this.parsedStmt; - queryStmt = parsedStmt.getSelectStmt(); + queryStmt = parsedStmt.getQueryStmt(); queryStmt.getTables(analyzer, tableMap, parentViewNameSet); } else { InsertStmt insertStmt = (InsertStmt) parsedStmt; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 7a752e6c0e4858..053a899fb551cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -125,7 +125,8 @@ public void testVarchar() throws Exception { createTableAsSelect(selectFromDecimal); ShowResultSet showResultSet = showCreateTable("select_varchar"); Assert.assertEquals("CREATE TABLE `select_varchar` (\n" + - " `userId` string NULL COMMENT \"\"\n" + + " `userId` string NULL COMMENT \"\",\n" + + " `username` string NULL COMMENT \"\"\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`userId`)\n" + "COMMENT \"OLAP\"\n" + @@ -142,6 +143,7 @@ public void testFunction() throws Exception { String selectFromFunction1 = "create table `test`.`select_function_1` PROPERTIES(\"replication_num\" = \"1\") as select count(*) from `test`.`varchar_table`;\n"; createTableAsSelect(selectFromFunction1); ShowResultSet showResultSet1 = showCreateTable("select_function_1"); + System.out.println(showResultSet1.getResultRows().get(0).get(1)); Assert.assertEquals("CREATE TABLE `select_function_1` (\n" + " `_col0` bigint(20) NULL COMMENT \"\"\n" + ") ENGINE=OLAP\n" + @@ -154,13 +156,17 @@ public void testFunction() throws Exception { "\"storage_format\" = \"V2\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); - String selectFromFunction2 = "create table `test`.`select_function_2` PROPERTIES(\"replication_num\" = \"1\") as select sum(status) from `test`.`join_table`;\n"; + String selectFromFunction2 = "create table `test`.`select_function_2` PROPERTIES(\"replication_num\" = \"1\") as select sum(status), sum(status), sum(status), count(status), count(status) from `test`.`join_table`;\n"; createTableAsSelect(selectFromFunction2); ShowResultSet showResultSet2 = showCreateTable("select_function_2"); Assert.assertEquals("CREATE TABLE `select_function_2` (\n" + - " `_col0` bigint(20) NULL COMMENT \"\"\n" + + " `_col0` bigint(20) NULL COMMENT \"\",\n" + + " `_col1` bigint(20) NULL COMMENT \"\",\n" + + " `_col2` bigint(20) NULL COMMENT \"\",\n" + + " `_col3` bigint(20) NULL COMMENT \"\",\n" + + " `_col4` bigint(20) NULL COMMENT \"\"\n" + ") ENGINE=OLAP\n" + - "DUPLICATE KEY(`_col0`)\n" + + "DUPLICATE KEY(`_col0`, `_col1`, `_col2`)\n" + "COMMENT \"OLAP\"\n" + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + @@ -172,7 +178,7 @@ public void testFunction() throws Exception { @Test public void testAlias() throws Exception { - String selectAlias1 = "create table `test`.`select_alias_1` PROPERTIES(\"replication_num\" = \"1\") as select count(*) as amount from `test`.`varchar_table`;\n"; + String selectAlias1 = "create table `test`.`select_alias_1` PROPERTIES(\"replication_num\" = \"1\") as select count(*) as amount from `test`.`varchar_table`"; createTableAsSelect(selectAlias1); ShowResultSet showResultSet1 = showCreateTable("select_alias_1"); Assert.assertEquals("CREATE TABLE `select_alias_1` (\n" + @@ -186,11 +192,12 @@ public void testAlias() throws Exception { "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); - String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") as select userId as alias_name from `test`.`varchar_table`;\n"; + String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") as select userId as alias_name, username from `test`.`varchar_table`;\n"; createTableAsSelect(selectAlias2); ShowResultSet showResultSet2 = showCreateTable("select_alias_2"); Assert.assertEquals("CREATE TABLE `select_alias_2` (\n" + - " `alias_name` string NULL COMMENT \"\"\n" + + " `alias_name` string NULL COMMENT \"\",\n" + + " `username` string NULL COMMENT \"\"\n" + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`alias_name`)\n" + "COMMENT \"OLAP\"\n" + @@ -243,4 +250,42 @@ public void testName() throws Exception { "\"storage_format\" = \"V2\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } + + @Test + public void testUnion() throws Exception { + String selectFromName = "create table `test`.`select_union` PROPERTIES(\"replication_num\" = \"1\") " + + "as select userId from `test`.`varchar_table` union select userId from `test`.`join_table`\n"; + createTableAsSelect(selectFromName); + ShowResultSet showResultSet = showCreateTable("select_union"); + Assert.assertEquals("CREATE TABLE `select_union` (\n" + + " `userId` string NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`userId`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");", showResultSet.getResultRows().get(0).get(1)); + } + + @Test + public void testCte() throws Exception { + String selectFromName = "create table `test`.`select_cte` PROPERTIES(\"replication_num\" = \"1\") " + + "as with cte_name1 as (select userId from `test`.`varchar_table`) select * from cte_name1"; + createTableAsSelect(selectFromName); + ShowResultSet showResultSet = showCreateTable("select_cte"); + Assert.assertEquals("CREATE TABLE `select_cte` (\n" + + " `userId` string NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`userId`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ");", showResultSet.getResultRows().get(0).get(1)); + } } From b491cf4d648411cb6c719bfbc70cc2077c566c15 Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 6 Sep 2021 22:50:08 +0800 Subject: [PATCH 08/10] MOD: unit test and analyze --- .../analysis/CreateTableAsSelectStmt.java | 3 + .../org/apache/doris/catalog/Catalog.java | 62 +++++++++---------- .../analysis/CreateTableAsSelectStmtTest.java | 23 ++++--- 3 files changed, 47 insertions(+), 41 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java index 3623cf6780736c..a5d17632f6a2aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java @@ -59,6 +59,9 @@ public void analyze(Analyzer analyzer) throws UserException { ErrorReport.reportAnalysisException(ErrorCode.ERR_UNSUPPORTED_TYPE_IN_CTAS, expr.getType()); } } + if (columnNames != null && columnNames.size() != resultExprs.size()) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_COL_NUMBER_NOT_MATCH); + } } public CreateTableStmt getCreateTableStmt() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 1d9173230b8390..926b5817dc294e 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -3106,39 +3106,35 @@ public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlExceptio ArrayList colLabels = queryStmt.getColLabels(); int size = resultExprs.size(); // Check columnNames - if (columnNames != null && columnNames.size() != size) { - ErrorReport.report(ErrorCode.ERR_COL_NUMBER_NOT_MATCH); - } else { - int colNameIndex = 0; - for (int i = 0; i < size; ++i) { - String name; - if (columnNames != null) { - // use custom column names - name = columnNames.get(i); - } else { - name = colLabels.get(i); - } - try { - FeNameFormat.checkColumnName(name); - } catch (AnalysisException exception) { - name = "_col" + (colNameIndex++); - } - TypeDef typeDef; - Expr resultExpr = resultExprs.get(i); - // varchar/char transfer to string - if (resultExpr.getType().isStringType()) { - typeDef = new TypeDef(Type.STRING); - } else { - typeDef = new TypeDef(resultExpr.getType()); - } - createTableStmt.addColumnDef(new ColumnDef(name, typeDef, false, - null, true, - new DefaultValue(false, null), - "")); - // set first column as default distribution - if (createTableStmt.getDistributionDesc() == null && i == 0) { - createTableStmt.setDistributionDesc(new HashDistributionDesc(10, Lists.newArrayList(name))); - } + int colNameIndex = 0; + for (int i = 0; i < size; ++i) { + String name; + if (columnNames != null) { + // use custom column names + name = columnNames.get(i); + } else { + name = colLabels.get(i); + } + try { + FeNameFormat.checkColumnName(name); + } catch (AnalysisException exception) { + name = "_col" + (colNameIndex++); + } + TypeDef typeDef; + Expr resultExpr = resultExprs.get(i); + // varchar/char transfer to string + if (resultExpr.getType().isStringType()) { + typeDef = new TypeDef(Type.STRING); + } else { + typeDef = new TypeDef(resultExpr.getType()); + } + createTableStmt.addColumnDef(new ColumnDef(name, typeDef, false, + null, true, + new DefaultValue(false, null), + "")); + // set first column as default distribution + if (createTableStmt.getDistributionDesc() == null && i == 0) { + createTableStmt.setDistributionDesc(new HashDistributionDesc(10, Lists.newArrayList(name))); } } Analyzer dummyRootAnalyzer = new Analyzer(this, ConnectContext.get()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 053a899fb551cf..fdae9733eb90b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -114,14 +114,21 @@ private static ShowResultSet showCreateTable(String tableName) throws Exception @Test public void testErrorType() throws Exception { - String selectFromDecimal = "create table `test`.`select_decimal_table` PROPERTIES(\"replication_num\" = \"1\") as select * from `test`.`decimal_table`;\n"; + String selectFromDecimal = "create table `test`.`select_decimal_table` PROPERTIES(\"replication_num\" = \"1\") as select * from `test`.`decimal_table`"; ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Unsupported type", () -> UtFrameUtils.parseAndAnalyzeStmt(selectFromDecimal, connectContext)); } + @Test + public void testErrorColumn() throws Exception { + String selectFromColumn = "create table `test`.`select_column_table`(test_error) PROPERTIES(\"replication_num\" = \"1\") as select * from `test`.`varchar_table`"; + ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Number of columns don't equal number of SELECT statement's select list", + () -> UtFrameUtils.parseAndAnalyzeStmt(selectFromColumn, connectContext)); + } + @Test public void testVarchar() throws Exception { - String selectFromDecimal = "create table `test`.`select_varchar` PROPERTIES(\"replication_num\" = \"1\") as select * from `test`.`varchar_table`;\n"; + String selectFromDecimal = "create table `test`.`select_varchar` PROPERTIES(\"replication_num\" = \"1\") as select * from `test`.`varchar_table`"; createTableAsSelect(selectFromDecimal); ShowResultSet showResultSet = showCreateTable("select_varchar"); Assert.assertEquals("CREATE TABLE `select_varchar` (\n" + @@ -140,7 +147,7 @@ public void testVarchar() throws Exception { @Test public void testFunction() throws Exception { - String selectFromFunction1 = "create table `test`.`select_function_1` PROPERTIES(\"replication_num\" = \"1\") as select count(*) from `test`.`varchar_table`;\n"; + String selectFromFunction1 = "create table `test`.`select_function_1` PROPERTIES(\"replication_num\" = \"1\") as select count(*) from `test`.`varchar_table`"; createTableAsSelect(selectFromFunction1); ShowResultSet showResultSet1 = showCreateTable("select_function_1"); System.out.println(showResultSet1.getResultRows().get(0).get(1)); @@ -156,7 +163,7 @@ public void testFunction() throws Exception { "\"storage_format\" = \"V2\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); - String selectFromFunction2 = "create table `test`.`select_function_2` PROPERTIES(\"replication_num\" = \"1\") as select sum(status), sum(status), sum(status), count(status), count(status) from `test`.`join_table`;\n"; + String selectFromFunction2 = "create table `test`.`select_function_2` PROPERTIES(\"replication_num\" = \"1\") as select sum(status), sum(status), sum(status), count(status), count(status) from `test`.`join_table`"; createTableAsSelect(selectFromFunction2); ShowResultSet showResultSet2 = showCreateTable("select_function_2"); Assert.assertEquals("CREATE TABLE `select_function_2` (\n" + @@ -192,7 +199,7 @@ public void testAlias() throws Exception { "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); - String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") as select userId as alias_name, username from `test`.`varchar_table`;\n"; + String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") as select userId as alias_name, username from `test`.`varchar_table`"; createTableAsSelect(selectAlias2); ShowResultSet showResultSet2 = showCreateTable("select_alias_2"); Assert.assertEquals("CREATE TABLE `select_alias_2` (\n" + @@ -212,7 +219,7 @@ public void testAlias() throws Exception { @Test public void testJoin() throws Exception { String selectFromJoin = "create table `test`.`select_join` PROPERTIES(\"replication_num\" = \"1\") " + - "as select vt.userId, vt.username, jt.status from `test`.`varchar_table` vt join `test`.`join_table` jt on vt.userId=jt.userId;\n"; + "as select vt.userId, vt.username, jt.status from `test`.`varchar_table` vt join `test`.`join_table` jt on vt.userId=jt.userId"; createTableAsSelect(selectFromJoin); ShowResultSet showResultSet = showCreateTable("select_join"); Assert.assertEquals("CREATE TABLE `select_join` (\n" + @@ -233,7 +240,7 @@ public void testJoin() throws Exception { @Test public void testName() throws Exception { String selectFromName = "create table `test`.`select_name`(user, testname, userstatus) PROPERTIES(\"replication_num\" = \"1\") " + - "as select vt.userId, vt.username, jt.status from `test`.`varchar_table` vt join `test`.`join_table` jt on vt.userId=jt.userId;\n"; + "as select vt.userId, vt.username, jt.status from `test`.`varchar_table` vt join `test`.`join_table` jt on vt.userId=jt.userId"; createTableAsSelect(selectFromName); ShowResultSet showResultSet = showCreateTable("select_name"); Assert.assertEquals("CREATE TABLE `select_name` (\n" + @@ -254,7 +261,7 @@ public void testName() throws Exception { @Test public void testUnion() throws Exception { String selectFromName = "create table `test`.`select_union` PROPERTIES(\"replication_num\" = \"1\") " + - "as select userId from `test`.`varchar_table` union select userId from `test`.`join_table`\n"; + "as select userId from `test`.`varchar_table` union select userId from `test`.`join_table`"; createTableAsSelect(selectFromName); ShowResultSet showResultSet = showCreateTable("select_union"); Assert.assertEquals("CREATE TABLE `select_union` (\n" + From 01d97c20589c5100fba0d69152cbaf1fa3b7c821 Mon Sep 17 00:00:00 2001 From: stalary Date: Mon, 6 Sep 2021 22:54:44 +0800 Subject: [PATCH 09/10] MOD: clone --- .../src/main/java/org/apache/doris/analysis/SelectStmt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index 0f10a287938eee..0f2167e789ad13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -188,7 +188,7 @@ public void reset() { } @Override - public SelectStmt clone() { + public QueryStmt clone() { return new SelectStmt(this); } From 53981225433316f21cbd0b683f23dc887ef717db Mon Sep 17 00:00:00 2001 From: stalary Date: Tue, 7 Sep 2021 09:35:33 +0800 Subject: [PATCH 10/10] MOD: clean --- fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index 926b5817dc294e..924a798b17d6da 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -85,7 +85,6 @@ import org.apache.doris.analysis.RollupRenameClause; import org.apache.doris.analysis.ShowAlterStmt.AlterType; import org.apache.doris.analysis.SinglePartitionDesc; -import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.TableRenameClause;