From d7a39473f0ee3a22e78d33655ee78a721e4fa217 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 13 Mar 2024 15:24:58 +0800 Subject: [PATCH 01/23] [feat](nereids) support create view in nereids --- .../org/apache/doris/nereids/DorisParser.g4 | 3 ++ .../nereids/parser/LogicalPlanBuilder.java | 15 ++++++++ .../doris/nereids/trees/plans/PlanType.java | 3 +- .../plans/commands/CreateViewCommand.java | 21 ++++++++++++ .../plans/commands/info/CreateViewInfo.java | 34 +++++++++++++++++++ 5 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 7aaf9fbde652c2..0fd377df874649 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -60,6 +60,9 @@ statementBase properties=propertyClause? (BROKER extProperties=propertyClause)? (AS query)? #createTable + | CREATE VIEW (IF NOT EXISTS)? name=multipartIdentifier + (LEFT_PAREN cols=simpleColumnDefs RIGHT_PAREN)? + (COMMENT STRING_LITERAL)? AS query #createView | explain? INSERT (INTO | OVERWRITE TABLE) (tableName=multipartIdentifier | DORIS_INTERNAL_TABLE_ID LEFT_PAREN tableId=INTEGER_VALUE RIGHT_PAREN) partitionSpec? // partition define diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 2985664f8b818a..bba4ee884845de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -77,6 +77,7 @@ import org.apache.doris.nereids.DorisParser.CreateProcedureContext; import org.apache.doris.nereids.DorisParser.CreateRowPolicyContext; import org.apache.doris.nereids.DorisParser.CreateTableContext; +import org.apache.doris.nereids.DorisParser.CreateViewContext; import org.apache.doris.nereids.DorisParser.CteContext; import org.apache.doris.nereids.DorisParser.DataTypeWithNullableContext; import org.apache.doris.nereids.DorisParser.DateCeilContext; @@ -360,6 +361,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateViewCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromUsingCommand; import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand; @@ -386,6 +388,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateViewInfo; import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; import org.apache.doris.nereids.trees.plans.commands.info.DefaultValue; import org.apache.doris.nereids.trees.plans.commands.info.DistributionDescriptor; @@ -2413,6 +2416,18 @@ private LogicalPlan plan(ParserRuleContext tree) { * create table parsing * ******************************************************************************************** */ + @Override + public LogicalPlan visitCreateView(CreateViewContext ctx) { + List nameParts = visitMultipartIdentifier(ctx.name); + String comment = ctx.STRING_LITERAL() == null ? "" : LogicalPlanBuilderAssistant.escapeBackSlash( + ctx.STRING_LITERAL().getText().substring(1, ctx.STRING_LITERAL().getText().length() - 1)); + LogicalPlan logicalPlan = visitQuery(ctx.query()); + String querySql = getOriginSql(ctx.query()); + CreateViewInfo info = new CreateViewInfo(ctx.EXISTS() != null, new TableNameInfo(nameParts), comment, logicalPlan, querySql, + ctx.cols == null ? Lists.newArrayList() : visitSimpleColumnDefs(ctx.cols)); + return new CreateViewCommand(info); + } + @Override public LogicalPlan visitCreateTable(CreateTableContext ctx) { String ctlName = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 1c493deae033cb..daab18438b4774 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -148,5 +148,6 @@ public enum PlanType { CREATE_PROCEDURE_COMMAND, DROP_PROCEDURE_COMMAND, SHOW_PROCEDURE_COMMAND, - SHOW_CREATE_PROCEDURE_COMMAND + SHOW_CREATE_PROCEDURE_COMMAND, + CREATE_VIEW_COMMAND } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java new file mode 100644 index 00000000000000..c057ea0302f2ed --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java @@ -0,0 +1,21 @@ +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.CreateViewInfo; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +public class CreateViewCommand extends Command { + private final CreateViewInfo createViewInfo; + + public CreateViewCommand(CreateViewInfo createViewInfo) { + super(PlanType.CREATE_VIEW_COMMAND); + this.createViewInfo = createViewInfo; + } + + @Override + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java new file mode 100644 index 00000000000000..e0df174c6bb2a3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -0,0 +1,34 @@ +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.CreateViewStmt; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import java.util.List; + +/** + * CreateViewInfo + */ +public class CreateViewInfo { + private final boolean ifNotExists; + private final TableNameInfo viewName; + private final String comment; + private final LogicalPlan logicalQuery; + private final String querySql; + + private final List simpleColumnDefinitions; + + /** constructor*/ + public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String comment, LogicalPlan logicalQuery, + String querySql, List simpleColumnDefinitions) { + this.ifNotExists = ifNotExists; + this.viewName = viewName; + this.comment = comment; + this.logicalQuery = logicalQuery; + this.querySql = querySql; + this.simpleColumnDefinitions = simpleColumnDefinitions; + } + + public CreateViewStmt translateToLegacyStmt() { + + } +} From 0f67d8f28d99c3c008c583b7faed8e2880034e7a Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 20 Mar 2024 19:35:14 +0800 Subject: [PATCH 02/23] [feat](nereids) support create view in nereids --- .../apache/doris/analysis/BaseViewStmt.java | 3 +- .../apache/doris/analysis/CreateViewStmt.java | 24 ++++++ .../doris/nereids/jobs/executor/Analyzer.java | 3 +- .../nereids/parser/LogicalPlanBuilder.java | 15 +++- .../nereids/rules/analysis/BindRelation.java | 32 +++++++- .../plans/commands/CreateViewCommand.java | 29 ++++++- .../plans/commands/info/CreateViewInfo.java | 77 ++++++++++++++++++- .../commands/info/SimpleColumnDefinition.java | 6 ++ .../trees/plans/visitor/CommandVisitor.java | 5 ++ 9 files changed, 185 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseViewStmt.java index 8114448f0d4e15..468a3de80245a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseViewStmt.java @@ -25,7 +25,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.ToSqlContext; -import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; @@ -50,7 +49,7 @@ public class BaseViewStmt extends DdlStmt { protected QueryStmt cloneStmt; public BaseViewStmt(TableName tableName, List cols, QueryStmt queryStmt) { - Preconditions.checkNotNull(queryStmt); + // Preconditions.checkNotNull(queryStmt); this.tableName = tableName; this.cols = cols; this.viewDefStmt = queryStmt; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java index ba5007ccce85cd..c7a6301580a5ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java @@ -17,6 +17,7 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; @@ -24,6 +25,8 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; @@ -92,4 +95,25 @@ public void analyze(Analyzer analyzer) throws UserException { } } } + + public void createColumnAndViewDefsForNereids(String querySql, Plan plan) { + List outputs = plan.getOutput(); + if (cols.isEmpty()) { + for (Slot output : outputs) { + Column column = new Column(output.getName(), output.getDataType().toCatalogDataType()); + finalCols.add(column); + } + } else { + for (int i = 0; i < cols.size(); ++i) { + Column column = new Column(cols.get(i).getColName(), outputs.get(i).getDataType().toCatalogDataType()); + column.setComment(cols.get(i).getComment()); + finalCols.add(column); + } + } + inlineViewDef = querySql; + } + + public List getFinalColumns() { + return finalCols; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index 9ad10a30aa29bd..90678f6c6ca5e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -120,7 +120,8 @@ private static List buildAnalyzeViewJobs(Optional tableQualifier, CascadesContext cascadesContext) { + List tableQualifier, CascadesContext cascadesContext) { switch (table.getType()) { case OLAP: case MATERIALIZED_VIEW: @@ -247,7 +251,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio String inlineViewDef = view.getInlineViewDef(); Plan viewBody = parseAndAnalyzeView(inlineViewDef, cascadesContext); LogicalView logicalView = new LogicalView<>(view, viewBody); - return new LogicalSubQueryAlias<>(tableQualifier, logicalView); + return new LogicalSubQueryAlias<>(tableQualifier, addProjectForView(view, logicalView)); case HMS_EXTERNAL_TABLE: HMSExternalTable hmsTable = (HMSExternalTable) table; if (Config.enable_query_hive_views && hmsTable.isView()) { @@ -328,6 +332,30 @@ private List getPartitionIds(TableIf t, UnboundRelation unboundRelation) { }).collect(ImmutableList.toImmutableList()); } + private LogicalPlan addProjectForView(View view, LogicalView logicalView) { + List viewOutput = logicalView.getOutput(); + List fullSchema = view.getFullSchema(); + boolean needProject = false; + for (int i = 0; i < viewOutput.size(); i++) { + if (!viewOutput.get(i).getName().equals(fullSchema.get(i).getName())) { + needProject = true; + break; + } + } + if (!needProject) { + return logicalView; + } + List projectList = Lists.newArrayList(); + for (int i = 0; i < viewOutput.size(); i++) { + if (viewOutput.get(i).getName().equals(fullSchema.get(i).getName())) { + projectList.add(viewOutput.get(i)); + } else { + projectList.add(new Alias(viewOutput.get(i), fullSchema.get(i).getName())); + } + } + return new LogicalProject(projectList, logicalView); + } + /** CustomTableResolver */ public interface CustomTableResolver extends Function, TableIf> {} } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java index c057ea0302f2ed..7476b1f039a101 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java @@ -1,10 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.analysis.CreateViewStmt; +import org.apache.doris.catalog.Env; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.info.CreateViewInfo; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; +/** CreateViewCommand */ public class CreateViewCommand extends Command { private final CreateViewInfo createViewInfo; @@ -15,7 +36,13 @@ public CreateViewCommand(CreateViewInfo createViewInfo) { @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { - + CreateViewStmt createViewStmt = createViewInfo.translateToLegacyStmt(ctx); + createViewInfo.validate(ctx); + Env.getCurrentEnv().createView(createViewStmt); } + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitCreateViewCommand(this, context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index e0df174c6bb2a3..c5aeca5fe716a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -1,20 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.trees.plans.commands.info; +import org.apache.doris.analysis.ColWithComment; import org.apache.doris.analysis.CreateViewStmt; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.Util; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.analyzer.UnboundResultSink; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import java.util.List; +import java.util.Set; /** * CreateViewInfo */ public class CreateViewInfo { + private CreateViewStmt createViewStmt; private final boolean ifNotExists; private final TableNameInfo viewName; private final String comment; private final LogicalPlan logicalQuery; private final String querySql; - private final List simpleColumnDefinitions; /** constructor*/ @@ -23,12 +61,47 @@ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String commen this.ifNotExists = ifNotExists; this.viewName = viewName; this.comment = comment; + if (logicalQuery instanceof LogicalFileSink) { + throw new AnalysisException("Not support OUTFILE clause in CREATE VIEW statement"); + } this.logicalQuery = logicalQuery; this.querySql = querySql; this.simpleColumnDefinitions = simpleColumnDefinitions; } - public CreateViewStmt translateToLegacyStmt() { + /**translateToLegacyStmt*/ + public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { + viewName.analyze(ctx); + List cols = Lists.newArrayList(); + for (SimpleColumnDefinition def : simpleColumnDefinitions) { + cols.add(def.transferToColWithComment()); + } + createViewStmt = new CreateViewStmt(ifNotExists, viewName.transferToTableName(), cols, comment, + null); + NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); + Plan plan = planner.plan(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE); + createViewStmt.createColumnAndViewDefsForNereids(querySql, plan); + return createViewStmt; + } + /**validate*/ + public void validate(ConnectContext ctx) throws UserException { + FeNameFormat.checkTableName(viewName.getTbl()); + // disallow external catalog + Util.prohibitExternalCatalog(viewName.getCtl(), "CreateViewStmt"); + // check privilege + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(), viewName.getDb(), + viewName.getTbl(), PrivPredicate.CREATE)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE"); + } + // Do not rewrite nondeterministic functions to constant in create view's def stmt + // ctx.setNotEvalNondeterministicFunction(true); + List finalCols = createViewStmt.getFinalColumns(); + Set colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + for (Column col : finalCols) { + if (!colSets.add(col.getName())) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, col.getName()); + } + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SimpleColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SimpleColumnDefinition.java index 51080bf45d2372..df6b88f6306701 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SimpleColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SimpleColumnDefinition.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.commands.info; +import org.apache.doris.analysis.ColWithComment; + /** * column def for mv */ @@ -39,4 +41,8 @@ public String getName() { public String getComment() { return comment; } + + public ColWithComment transferToColWithComment() { + return new ColWithComment(name, comment); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index 6bcc41e39e45f9..527ccb1ffe080b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateProcedureCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateViewCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromUsingCommand; import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand; @@ -156,4 +157,8 @@ default R visitShowProcedureStatusCommand(ShowProcedureStatusCommand showProcedu default R visitShowCreateProcedureCommand(ShowCreateProcedureCommand showCreateProcedureCommand, C context) { return visitCommand(showCreateProcedureCommand, context); } + + default R visitCreateViewCommand(CreateViewCommand createViewCommand, C context) { + return visitCommand(createViewCommand, context); + } } From fa35f3ff3a6e89ce0d9f6851e520174a76ae4e99 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 21 Mar 2024 20:46:51 +0800 Subject: [PATCH 03/23] [feat](nereids) support create view in nereids --- .../apache/doris/analysis/CreateViewStmt.java | 9 +- .../doris/nereids/analyzer/UnboundStar.java | 14 +- .../nereids/parser/LogicalPlanBuilder.java | 13 +- .../doris/nereids/parser/NereidsParser.java | 3 +- .../rules/analysis/BindExpression.java | 6 +- .../rules/analysis/ExpressionAnalyzer.java | 2 +- .../nereids/trees/expressions/BoundStar.java | 13 ++ .../trees/expressions/SlotReference.java | 2 +- .../plans/commands/CreateViewCommand.java | 2 +- .../plans/commands/info/CreateViewInfo.java | 182 ++++++++++++++++-- .../trees/plans/logical/LogicalProject.java | 10 + .../suites/ddl_p0/test_create_view.groovy | 3 + 12 files changed, 221 insertions(+), 38 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java index c7a6301580a5ec..b85fb50245422c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java @@ -96,7 +96,7 @@ public void analyze(Analyzer analyzer) throws UserException { } } - public void createColumnAndViewDefsForNereids(String querySql, Plan plan) { + public void createColumnAndViewDefsForNereids(Plan plan) { List outputs = plan.getOutput(); if (cols.isEmpty()) { for (Slot output : outputs) { @@ -110,10 +110,13 @@ public void createColumnAndViewDefsForNereids(String querySql, Plan plan) { finalCols.add(column); } } + } + + public void setInlineViewDef(String querySql) { inlineViewDef = querySql; } - public List getFinalColumns() { - return finalCols; + public void setFinalColumns(List columns) { + finalCols.addAll(columns); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java index 43705611562b71..ce22be6dffe257 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; +import lombok.Getter; import java.util.List; import java.util.Objects; @@ -33,14 +34,21 @@ * Star expression. */ public class UnboundStar extends NamedExpression implements LeafExpression, Unbound, PropagateNullable { - private final List qualifier; + private int indexInSqlString; + public UnboundStar(List qualifier) { super(ImmutableList.of()); this.qualifier = Objects.requireNonNull(ImmutableList.copyOf(qualifier), "qualifier can not be null"); } + public UnboundStar(List qualifier, int indexInSqlString) { + super(ImmutableList.of()); + this.qualifier = Objects.requireNonNull(ImmutableList.copyOf(qualifier), "qualifier can not be null"); + this.indexInSqlString = indexInSqlString; + } + @Override public String toSql() { return Utils.qualifiedName(qualifier, "*"); @@ -71,6 +79,10 @@ public boolean equals(Object o) { return qualifier.equals(that.qualifier); } + public int getIndexInSqlString() { + return indexInSqlString; + } + @Override public int hashCode() { return Objects.hash(super.hashCode(), qualifier); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 1572e4ed655384..b8f8cd5c5c5d7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1246,21 +1246,12 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte LogicalPlan relation; if (ctx.fromClause() == null) { SelectColumnClauseContext columnCtx = selectCtx.selectColumnClause(); - String sql = getOriginSql(columnCtx); - int startIndex = columnCtx.start.getStartIndex(); - int stopIndex = columnCtx.stop.getStopIndex(); - System.out.println(sql + ' ' + startIndex + ' ' + stopIndex); if (columnCtx.EXCEPT() != null) { throw new ParseException("select-except cannot be used in one row relation", selectCtx); } relation = new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), ImmutableList.of(new UnboundAlias(Literal.of(0)))); } else { - SelectColumnClauseContext columnCtx = selectCtx.selectColumnClause(); - String sql = getOriginSql(columnCtx); - int startIndex = columnCtx.start.getStartIndex(); - int stopIndex = columnCtx.stop.getStopIndex(); - System.out.println(sql + ' ' + startIndex + ' ' + stopIndex); relation = visitFromClause(ctx.fromClause()); } if (ctx.intoClause() != null && !ConnectContext.get().isRunProcedure()) { @@ -1395,9 +1386,7 @@ public Expression visitStar(StarContext ctx) { target = ImmutableList.of(); } int startIndex = ctx.start.getStartIndex(); - int stopIndex = ctx.stop.getStopIndex(); - System.out.println(startIndex + " " + stopIndex); - return new UnboundStar(target); + return new UnboundStar(target, startIndex); }); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 0ee3f5d068f9ce..d40b45d41a4042 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -164,7 +164,8 @@ private T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder, return (T) realLogicalPlanBuilder.visit(tree); } - private ParserRuleContext toAst(String sql, Function parseFunction) { + /** toAst */ + public static ParserRuleContext toAst(String sql, Function parseFunction) { DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); CommonTokenStream tokenStream = new CommonTokenStream(lexer); DorisParser parser = new DorisParser(tokenStream); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index c2c7f5815d9288..f7989743d4c840 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -567,12 +567,14 @@ private Plan bindProject(MatchingContext> ctx) { () -> analyzer.analyzeToSet(project.getExcepts())); Builder boundProjections = ImmutableList.builderWithExpectedSize(project.arity()); + List boundStars = Lists.newArrayList(); for (Expression expression : project.getProjects()) { Expression expr = analyzer.analyze(expression); if (!(expr instanceof BoundStar)) { boundProjections.add((NamedExpression) expr); } else { BoundStar boundStar = (BoundStar) expr; + boundStars.add(boundStar); List slots = boundStar.getSlots(); if (!excepts.isEmpty()) { slots = Utils.filterImmutableList(slots, slot -> !boundExcepts.get().contains(slot)); @@ -580,7 +582,9 @@ private Plan bindProject(MatchingContext> ctx) { boundProjections.addAll(slots); } } - return project.withProjects(boundProjections.build()); + LogicalProject finalProject = project.withProjects(boundProjections.build()); + finalProject.setBoundStars(boundStars); + return finalProject; } private Plan bindFilter(MatchingContext> ctx) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index f4c1b428d4147b..43113ad2047e12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -275,7 +275,7 @@ public Expression visitUnboundStar(UnboundStar unboundStar, ExpressionRewriteCon .collect(Collectors.toList()); switch (qualifier.size()) { case 0: // select * - return new BoundStar(slots); + return new BoundStar(slots, unboundStar.getIndexInSqlString()); case 1: // select table.* case 2: // select db.table.* case 3: // select catalog.db.table.* diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java index 8b4bffad3fc817..5bab4aeeb49362 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java @@ -28,6 +28,8 @@ /** BoundStar is used to wrap list of slots for temporary. */ public class BoundStar extends NamedExpression implements PropagateNullable { + private int indexInSqlString; + public BoundStar(List children) { super((List) children); Preconditions.checkArgument(children.stream().noneMatch(slot -> slot instanceof UnboundSlot), @@ -35,6 +37,14 @@ public BoundStar(List children) { ); } + public BoundStar(List children, int indexInSqlString) { + super((List) children); + Preconditions.checkArgument(children.stream().noneMatch(slot -> slot instanceof UnboundSlot), + "BoundStar can not wrap UnboundSlot" + ); + this.indexInSqlString = indexInSqlString; + } + public String toSql() { return children.stream().map(Expression::toSql).collect(Collectors.joining(", ")); } @@ -44,6 +54,9 @@ public List getSlots() { return (List) children(); } + public int getIndexInSqlString() { + return indexInSqlString; + } @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitBoundStar(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 7cfaad72a2c546..80e5b057fb3ea9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -187,7 +187,7 @@ public Optional getTable() { @Override public String toSql() { - return name.get(); + return '`' + name.get() + '`'; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java index 7476b1f039a101..8a31c869348fda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java @@ -36,8 +36,8 @@ public CreateViewCommand(CreateViewInfo createViewInfo) { @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { - CreateViewStmt createViewStmt = createViewInfo.translateToLegacyStmt(ctx); createViewInfo.validate(ctx); + CreateViewStmt createViewStmt = createViewInfo.translateToLegacyStmt(ctx); Env.getCurrentEnv().createView(createViewStmt); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index c5aeca5fe716a8..ceb5da22329fdf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -24,36 +24,52 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.DorisParser; +import org.apache.doris.nereids.DorisParserBaseVisitor; import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.BoundStar; +import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.RuleNode; +import org.apache.commons.lang3.StringUtils; import java.util.List; +import java.util.Map; import java.util.Set; /** * CreateViewInfo */ public class CreateViewInfo { - private CreateViewStmt createViewStmt; private final boolean ifNotExists; private final TableNameInfo viewName; private final String comment; private final LogicalPlan logicalQuery; private final String querySql; private final List simpleColumnDefinitions; + private final List finalCols = Lists.newArrayList(); + private Plan plan; /** constructor*/ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String comment, LogicalPlan logicalQuery, @@ -69,23 +85,10 @@ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String commen this.simpleColumnDefinitions = simpleColumnDefinitions; } - /**translateToLegacyStmt*/ - public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { - viewName.analyze(ctx); - List cols = Lists.newArrayList(); - for (SimpleColumnDefinition def : simpleColumnDefinitions) { - cols.add(def.transferToColWithComment()); - } - createViewStmt = new CreateViewStmt(ifNotExists, viewName.transferToTableName(), cols, comment, - null); - NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - Plan plan = planner.plan(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE); - createViewStmt.createColumnAndViewDefsForNereids(querySql, plan); - return createViewStmt; - } - /**validate*/ public void validate(ConnectContext ctx) throws UserException { + NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); + plan = planner.plan(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE); FeNameFormat.checkTableName(viewName.getTbl()); // disallow external catalog Util.prohibitExternalCatalog(viewName.getCtl(), "CreateViewStmt"); @@ -94,9 +97,9 @@ public void validate(ConnectContext ctx) throws UserException { viewName.getTbl(), PrivPredicate.CREATE)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE"); } + createFinalCols(); // Do not rewrite nondeterministic functions to constant in create view's def stmt // ctx.setNotEvalNondeterministicFunction(true); - List finalCols = createViewStmt.getFinalColumns(); Set colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (Column col : finalCols) { if (!colSets.add(col.getName())) { @@ -104,4 +107,149 @@ public void validate(ConnectContext ctx) throws UserException { } } } + + /**translateToLegacyStmt*/ + public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { + List cols = Lists.newArrayList(); + for (SimpleColumnDefinition def : simpleColumnDefinitions) { + cols.add(def.transferToColWithComment()); + } + viewName.analyze(ctx); + CreateViewStmt createViewStmt = new CreateViewStmt(ifNotExists, viewName.transferToTableName(), cols, comment, + null); + // expand star(*) in project list + Plan analyzedPlan = parseAndAnalyzeView(querySql, ctx); + Map indexStringSqlMap = collectBoundStar(analyzedPlan); + String resSql = rewriteStarToColumn(indexStringSqlMap); + + // rewrite the project alias if defined + if (!simpleColumnDefinitions.isEmpty()) { + IndexFinder finder = new IndexFinder(); + ParserRuleContext tree = NereidsParser.toAst(resSql, DorisParser::singleStatement); + finder.visit(tree); + resSql = rewriteProjectsToUserDefineAlias(plan, resSql, finder); + } + createViewStmt.setFinalColumns(finalCols); + createViewStmt.setInlineViewDef(resSql); + return createViewStmt; + } + + private Plan parseAndAnalyzeView(String ddlSql, ConnectContext ctx) { + StatementContext stmtCtx = ctx.getStatementContext(); + LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(ddlSql); + // TODO: use a good to do this, such as eliminate UnboundResultSink + if (parsedViewPlan instanceof UnboundResultSink) { + parsedViewPlan = (LogicalPlan) ((UnboundResultSink) parsedViewPlan).child(); + } + CascadesContext viewContext = CascadesContext.initContext( + stmtCtx, parsedViewPlan, PhysicalProperties.ANY); + viewContext.keepOrShowPlanProcess(false, () -> { + viewContext.newAnalyzer(true).analyze(); + }); + return viewContext.getRewritePlan(); + } + + private String rewriteStarToColumn(Map indexStringSqlMap) { + StringBuilder builder = new StringBuilder(); + int beg = 0; + for (Map.Entry entry : indexStringSqlMap.entrySet()) { + int index = entry.getKey(); + builder.append(querySql, beg, index); + builder.append(entry.getValue()); + beg = index + 1; + } + builder.append(querySql, beg, querySql.length()); + return builder.toString(); + } + + private String rewriteProjectsToUserDefineAlias(Plan plan, String resSql, + IndexFinder finder) { + List slots = plan.getOutput(); + StringBuilder replaceWithColsBuilder = new StringBuilder(); + for (int i = 0; i < slots.size(); ++i) { + replaceWithColsBuilder.append(slots.get(i).getName()); + replaceWithColsBuilder.append(" AS `"); + replaceWithColsBuilder.append(simpleColumnDefinitions.get(i).getName()); + replaceWithColsBuilder.append('`'); + if (i != slots.size() - 1) { + replaceWithColsBuilder.append(", "); + } + } + String replaceWithCols = replaceWithColsBuilder.toString(); + return StringUtils.overlay(resSql, replaceWithCols, finder.getIndex().first, + finder.getIndex().second + 1); + } + + private void createFinalCols() throws org.apache.doris.common.AnalysisException { + List outputs = plan.getOutput(); + if (simpleColumnDefinitions.isEmpty()) { + for (Slot output : outputs) { + Column column = new Column(output.getName(), output.getDataType().toCatalogDataType()); + finalCols.add(column); + } + } else { + if (outputs.size() != simpleColumnDefinitions.size()) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_VIEW_WRONG_LIST); + } + for (int i = 0; i < simpleColumnDefinitions.size(); ++i) { + Column column = new Column(simpleColumnDefinitions.get(i).getName(), + outputs.get(i).getDataType().toCatalogDataType()); + column.setComment(simpleColumnDefinitions.get(i).getComment()); + finalCols.add(column); + } + } + } + + private Map collectBoundStar(Plan plan) { + Map result = Maps.newTreeMap(); + plan.foreach(node -> { + if (node instanceof LogicalProject) { + LogicalProject project = (LogicalProject) node; + for (BoundStar star : project.getBoundStars()) { + result.put(star.getIndexInSqlString(), star.toSql()); + } + } + }); + return result; + } + + /** traverse ast to find the outermost project list location information in sql*/ + private static class IndexFinder extends DorisParserBaseVisitor { + private boolean found = false; + private int startIndex; + private int stopIndex; + + @Override + public Void visitChildren(RuleNode node) { + if (found) { + return null; + } + int n = node.getChildCount(); + for (int i = 0; i < n; ++i) { + ParseTree c = node.getChild(i); + c.accept(this); + } + return null; + } + + @Override + public Void visitCte(DorisParser.CteContext ctx) { + return null; + } + + @Override + public Void visitSelectColumnClause(DorisParser.SelectColumnClauseContext ctx) { + if (found) { + return null; + } + startIndex = ctx.getStart().getStartIndex(); + stopIndex = ctx.getStop().getStopIndex(); + found = true; + return null; + } + + public Pair getIndex() { + return Pair.of(startIndex, stopIndex); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index d899d228fb66bc..4e19c5ca623efd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -40,6 +40,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import org.json.JSONObject; import java.util.HashSet; @@ -58,6 +59,7 @@ public class LogicalProject extends LogicalUnary excepts; private final boolean isDistinct; private final boolean canEliminate; + private List boundStars = Lists.newArrayList(); public LogicalProject(List projects, CHILD_TYPE child) { this(projects, ImmutableList.of(), false, true, ImmutableList.of(child)); @@ -265,4 +267,12 @@ public ImmutableSet computeFdItems(Supplier> outputSupplier) return builder.build(); } + + public List getBoundStars() { + return boundStars; + } + + public void setBoundStars(List boundStars) { + this.boundStars = boundStars; + } } diff --git a/regression-test/suites/ddl_p0/test_create_view.groovy b/regression-test/suites/ddl_p0/test_create_view.groovy index 6020aa310509dd..d3cdbdf8171839 100644 --- a/regression-test/suites/ddl_p0/test_create_view.groovy +++ b/regression-test/suites/ddl_p0/test_create_view.groovy @@ -204,3 +204,6 @@ suite("test_create_view") { """ qt_test_view_6 """ SHOW VIEW FROM view_column_name_test;""" } +/* ApplicationName=DBeaver 23.3.5 - SQLEditor */ select * from mal_test1 + +LIMIT 0, 200 \ No newline at end of file From ffd501f23f129540054ea06f0c394ad9048fc98e Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 22 Mar 2024 16:15:50 +0800 Subject: [PATCH 04/23] [feat](nereids) support create view in nereids --- .../doris/nereids/analyzer/UnboundStar.java | 1 - .../nereids/trees/expressions/BoundStar.java | 5 ++ .../trees/expressions/SlotReference.java | 7 +- .../plans/commands/info/CreateViewInfo.java | 74 ++++++++++++------- .../org/apache/doris/nereids/util/Utils.java | 10 +++ .../suites/ddl_p0/test_create_view.groovy | 8 +- 6 files changed, 73 insertions(+), 32 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java index ce22be6dffe257..9645bad6038e5d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; -import lombok.Getter; import java.util.List; import java.util.Objects; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java index 5bab4aeeb49362..de67e433bfab4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java @@ -49,6 +49,11 @@ public String toSql() { return children.stream().map(Expression::toSql).collect(Collectors.joining(", ")); } + public String toSqlWithBacktick() { + return children.stream().map(slot -> ((SlotReference) slot).getQualifiedNameWithBacktick()) + .collect(Collectors.joining(", ")); + } + @SuppressWarnings({"unchecked", "rawtypes"}) public List getSlots() { return (List) children(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 80e5b057fb3ea9..34e315c99463e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.types.DataType; @@ -187,7 +188,7 @@ public Optional getTable() { @Override public String toSql() { - return '`' + name.get() + '`'; + return getQualifiedName(); } @Override @@ -293,4 +294,8 @@ private static Supplier> buildInternalName( internalName.isPresent() ? internalName : Optional.of(name.get())); } } + + public String getQualifiedNameWithBacktick() throws UnboundException { + return Utils.qualifiedNameWithBacktick(getQualifier(), getName()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index ceb5da22329fdf..472e77b42e932c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -39,11 +39,14 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.BoundStar; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Lists; @@ -57,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; /** * CreateViewInfo @@ -69,7 +73,8 @@ public class CreateViewInfo { private final String querySql; private final List simpleColumnDefinitions; private final List finalCols = Lists.newArrayList(); - private Plan plan; + private final List outputs = Lists.newArrayList(); + private Plan analyzedPlan; /** constructor*/ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String comment, LogicalPlan logicalQuery, @@ -88,7 +93,8 @@ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String commen /**validate*/ public void validate(ConnectContext ctx) throws UserException { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - plan = planner.plan(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE); + planner.plan(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE); + viewName.analyze(ctx); FeNameFormat.checkTableName(viewName.getTbl()); // disallow external catalog Util.prohibitExternalCatalog(viewName.getCtl(), "CreateViewStmt"); @@ -97,9 +103,14 @@ public void validate(ConnectContext ctx) throws UserException { viewName.getTbl(), PrivPredicate.CREATE)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE"); } + + analyzedPlan = parseAndAnalyzeView(querySql, ctx); + OutermostProjectFinder projectPlanFinder = new OutermostProjectFinder(); + AtomicReference> outermostProject = new AtomicReference<>(); + analyzedPlan.accept(projectPlanFinder, outermostProject); + outputs.addAll(outermostProject.get().getOutput()); + createFinalCols(); - // Do not rewrite nondeterministic functions to constant in create view's def stmt - // ctx.setNotEvalNondeterministicFunction(true); Set colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (Column col : finalCols) { if (!colSets.add(col.getName())) { @@ -114,23 +125,17 @@ public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { for (SimpleColumnDefinition def : simpleColumnDefinitions) { cols.add(def.transferToColWithComment()); } - viewName.analyze(ctx); CreateViewStmt createViewStmt = new CreateViewStmt(ifNotExists, viewName.transferToTableName(), cols, comment, null); // expand star(*) in project list - Plan analyzedPlan = parseAndAnalyzeView(querySql, ctx); Map indexStringSqlMap = collectBoundStar(analyzedPlan); - String resSql = rewriteStarToColumn(indexStringSqlMap); - - // rewrite the project alias if defined - if (!simpleColumnDefinitions.isEmpty()) { - IndexFinder finder = new IndexFinder(); - ParserRuleContext tree = NereidsParser.toAst(resSql, DorisParser::singleStatement); - finder.visit(tree); - resSql = rewriteProjectsToUserDefineAlias(plan, resSql, finder); - } + String rewrittenSql = rewriteStarToColumn(indexStringSqlMap); + + // rewrite project alias + rewrittenSql = rewriteProjectsToUserDefineAlias(rewrittenSql); + + createViewStmt.setInlineViewDef(rewrittenSql); createViewStmt.setFinalColumns(finalCols); - createViewStmt.setInlineViewDef(resSql); return createViewStmt; } @@ -162,16 +167,17 @@ private String rewriteStarToColumn(Map indexStringSqlMap) { return builder.toString(); } - private String rewriteProjectsToUserDefineAlias(Plan plan, String resSql, - IndexFinder finder) { - List slots = plan.getOutput(); + private String rewriteProjectsToUserDefineAlias(String resSql) { + IndexFinder finder = new IndexFinder(); + ParserRuleContext tree = NereidsParser.toAst(resSql, DorisParser::singleStatement); + finder.visit(tree); StringBuilder replaceWithColsBuilder = new StringBuilder(); - for (int i = 0; i < slots.size(); ++i) { - replaceWithColsBuilder.append(slots.get(i).getName()); + for (int i = 0; i < outputs.size(); ++i) { + replaceWithColsBuilder.append(((SlotReference) outputs.get(i)).getQualifiedNameWithBacktick()); replaceWithColsBuilder.append(" AS `"); - replaceWithColsBuilder.append(simpleColumnDefinitions.get(i).getName()); + replaceWithColsBuilder.append(finalCols.get(i).getName()); replaceWithColsBuilder.append('`'); - if (i != slots.size() - 1) { + if (i != outputs.size() - 1) { replaceWithColsBuilder.append(", "); } } @@ -181,7 +187,6 @@ private String rewriteProjectsToUserDefineAlias(Plan plan, String resSql, } private void createFinalCols() throws org.apache.doris.common.AnalysisException { - List outputs = plan.getOutput(); if (simpleColumnDefinitions.isEmpty()) { for (Slot output : outputs) { Column column = new Column(output.getName(), output.getDataType().toCatalogDataType()); @@ -206,13 +211,32 @@ private Map collectBoundStar(Plan plan) { if (node instanceof LogicalProject) { LogicalProject project = (LogicalProject) node; for (BoundStar star : project.getBoundStars()) { - result.put(star.getIndexInSqlString(), star.toSql()); + result.put(star.getIndexInSqlString(), star.toSqlWithBacktick()); } } }); return result; } + private static class OutermostProjectFinder extends DefaultPlanVisitor>> { + boolean found = false; + + @Override + public Void visit(Plan plan, AtomicReference> target) { + if (found) { + return null; + } + if (plan instanceof LogicalCTEProducer) { + return null; + } else if (plan instanceof LogicalProject) { + target.set((LogicalProject) plan); + found = true; + } + return super.visit(plan, target); + } + } + /** traverse ast to find the outermost project list location information in sql*/ private static class IndexFinder extends DorisParserBaseVisitor { private boolean found = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java index df9528cc49e2d6..5bf976d904ecff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java @@ -125,6 +125,16 @@ public static String qualifiedName(List qualifier, String name) { return StringUtils.join(qualifiedNameParts(qualifier, name), "."); } + /** get qualified name with Backtick */ + public static String qualifiedNameWithBacktick(List qualifiers, String name) { + List qualifierWithBacktick = Lists.newArrayListWithCapacity(qualifiers.size()); + for (String qualifier : qualifiers) { + qualifierWithBacktick.add('`' + qualifier + '`'); + } + String nameWithBacktick = '`' + name + '`'; + return StringUtils.join(qualifiedNameParts(qualifierWithBacktick, nameWithBacktick), "."); + } + /** * Get sql string for plan. * diff --git a/regression-test/suites/ddl_p0/test_create_view.groovy b/regression-test/suites/ddl_p0/test_create_view.groovy index d3cdbdf8171839..4b93c879101095 100644 --- a/regression-test/suites/ddl_p0/test_create_view.groovy +++ b/regression-test/suites/ddl_p0/test_create_view.groovy @@ -16,7 +16,8 @@ // under the License. suite("test_create_view") { - + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" sql """DROP TABLE IF EXISTS count_distinct""" sql """ CREATE TABLE IF NOT EXISTS count_distinct @@ -203,7 +204,4 @@ suite("test_create_view") { view_column_name_test """ qt_test_view_6 """ SHOW VIEW FROM view_column_name_test;""" -} -/* ApplicationName=DBeaver 23.3.5 - SQLEditor */ select * from mal_test1 - -LIMIT 0, 200 \ No newline at end of file +} \ No newline at end of file From 2f2bc3024f028e1202d6e60917b25f7c26141198 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sat, 23 Mar 2024 18:58:07 +0800 Subject: [PATCH 05/23] [feat](nereids) support create view in nereids --- .../doris/nereids/jobs/executor/Analyzer.java | 3 ++- .../nereids/trees/expressions/BoundStar.java | 1 + .../trees/expressions/SlotReference.java | 2 +- .../plans/commands/info/CreateViewInfo.java | 23 +++++++++++++++---- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index 90678f6c6ca5e6..ffff22e7fb97b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -121,7 +121,8 @@ private static List buildAnalyzeViewJobs(Optional getSlots() { public int getIndexInSqlString() { return indexInSqlString; } + @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitBoundStar(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 34e315c99463e5..989b4da08b4d72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -188,7 +188,7 @@ public Optional getTable() { @Override public String toSql() { - return getQualifiedName(); + return getQualifiedNameWithBacktick(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index 472e77b42e932c..06115ee2778b52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -37,7 +37,10 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.BoundStar; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; @@ -74,6 +77,7 @@ public class CreateViewInfo { private final List simpleColumnDefinitions; private final List finalCols = Lists.newArrayList(); private final List outputs = Lists.newArrayList(); + private final List projects = Lists.newArrayList(); private Plan analyzedPlan; /** constructor*/ @@ -109,7 +113,7 @@ public void validate(ConnectContext ctx) throws UserException { AtomicReference> outermostProject = new AtomicReference<>(); analyzedPlan.accept(projectPlanFinder, outermostProject); outputs.addAll(outermostProject.get().getOutput()); - + projects.addAll(outermostProject.get().getProjects()); createFinalCols(); Set colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (Column col : finalCols) { @@ -168,16 +172,25 @@ private String rewriteStarToColumn(Map indexStringSqlMap) { } private String rewriteProjectsToUserDefineAlias(String resSql) { + List projectExprs = Lists.newArrayList(); + for (int i = 0; i < projects.size(); i++) { + NamedExpression namedExpression = projects.get(i); + if (namedExpression instanceof Alias) { + projectExprs.add(namedExpression.child(0)); + } else { + projectExprs.add(namedExpression); + } + } IndexFinder finder = new IndexFinder(); ParserRuleContext tree = NereidsParser.toAst(resSql, DorisParser::singleStatement); finder.visit(tree); StringBuilder replaceWithColsBuilder = new StringBuilder(); - for (int i = 0; i < outputs.size(); ++i) { - replaceWithColsBuilder.append(((SlotReference) outputs.get(i)).getQualifiedNameWithBacktick()); + for (int i = 0; i < projectExprs.size(); ++i) { + replaceWithColsBuilder.append(projectExprs.get(i).toSql()); replaceWithColsBuilder.append(" AS `"); replaceWithColsBuilder.append(finalCols.get(i).getName()); replaceWithColsBuilder.append('`'); - if (i != outputs.size() - 1) { + if (i != projectExprs.size() - 1) { replaceWithColsBuilder.append(", "); } } @@ -211,7 +224,7 @@ private Map collectBoundStar(Plan plan) { if (node instanceof LogicalProject) { LogicalProject project = (LogicalProject) node; for (BoundStar star : project.getBoundStars()) { - result.put(star.getIndexInSqlString(), star.toSqlWithBacktick()); + result.put(star.getIndexInSqlString(), star.toSql()); } } }); From b5ebeeb7a15d1297924ca7bbc4c1cfc7382dd1d4 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sun, 24 Mar 2024 00:11:36 +0800 Subject: [PATCH 06/23] [feat](nereids) support create view in nereids --- .../doris/nereids/analyzer/UnboundStar.java | 7 ++++--- .../nereids/parser/LogicalPlanBuilder.java | 3 +-- .../rules/analysis/ExpressionAnalyzer.java | 8 +++++--- .../nereids/trees/expressions/BoundStar.java | 7 ++++--- .../expressions/functions/scalar/Lambda.java | 2 +- .../plans/commands/info/CreateViewInfo.java | 18 +++++++++--------- 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java index 9645bad6038e5d..14cfaffeb47fea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.analyzer; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; @@ -35,14 +36,14 @@ public class UnboundStar extends NamedExpression implements LeafExpression, Unbound, PropagateNullable { private final List qualifier; - private int indexInSqlString; + private Pair indexInSqlString; public UnboundStar(List qualifier) { super(ImmutableList.of()); this.qualifier = Objects.requireNonNull(ImmutableList.copyOf(qualifier), "qualifier can not be null"); } - public UnboundStar(List qualifier, int indexInSqlString) { + public UnboundStar(List qualifier, Pair indexInSqlString) { super(ImmutableList.of()); this.qualifier = Objects.requireNonNull(ImmutableList.copyOf(qualifier), "qualifier can not be null"); this.indexInSqlString = indexInSqlString; @@ -78,7 +79,7 @@ public boolean equals(Object o) { return qualifier.equals(that.qualifier); } - public int getIndexInSqlString() { + public Pair getIndexInSqlString() { return indexInSqlString; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index b8f8cd5c5c5d7b..7e463d2e3f6524 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1385,8 +1385,7 @@ public Expression visitStar(StarContext ctx) { } else { target = ImmutableList.of(); } - int startIndex = ctx.start.getStartIndex(); - return new UnboundStar(target, startIndex); + return new UnboundStar(target, Pair.of(ctx.start.getStartIndex(), ctx.stop.getStopIndex())); }); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index 43113ad2047e12..f099060d6cacbc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FunctionRegistry; import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; import org.apache.doris.common.util.Util; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -279,7 +280,7 @@ public Expression visitUnboundStar(UnboundStar unboundStar, ExpressionRewriteCon case 1: // select table.* case 2: // select db.table.* case 3: // select catalog.db.table.* - return bindQualifiedStar(qualifier, slots); + return bindQualifiedStar(qualifier, slots, unboundStar.getIndexInSqlString()); default: throw new AnalysisException("Not supported qualifier: " + StringUtils.join(qualifier, ".")); @@ -575,7 +576,8 @@ public Expression visitCast(Cast cast, ExpressionRewriteContext context) { return cast; } - private BoundStar bindQualifiedStar(List qualifierStar, List boundSlots) { + private BoundStar bindQualifiedStar(List qualifierStar, List boundSlots, + Pair indexInSql) { // FIXME: compatible with previous behavior: // https://github.com/apache/doris/pull/10415/files/3fe9cb0c3f805ab3a9678033b281b16ad93ec60a#r910239452 List slots = boundSlots.stream().filter(boundSlot -> { @@ -639,7 +641,7 @@ && compareDbName(qualifierStar.get(1), boundSlotQualifier.get(1)) if (slots.isEmpty()) { throw new AnalysisException("unknown qualifier: " + StringUtils.join(qualifierStar, ".") + ".*"); } - return new BoundStar(slots); + return new BoundStar(slots, indexInSql); } protected List bindSlotByThisScope(UnboundSlot unboundSlot) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java index ae4701d6cd75a3..6dec2f5debcf39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.expressions; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -28,7 +29,7 @@ /** BoundStar is used to wrap list of slots for temporary. */ public class BoundStar extends NamedExpression implements PropagateNullable { - private int indexInSqlString; + private Pair indexInSqlString; public BoundStar(List children) { super((List) children); @@ -37,7 +38,7 @@ public BoundStar(List children) { ); } - public BoundStar(List children, int indexInSqlString) { + public BoundStar(List children, Pair indexInSqlString) { super((List) children); Preconditions.checkArgument(children.stream().noneMatch(slot -> slot instanceof UnboundSlot), "BoundStar can not wrap UnboundSlot" @@ -59,7 +60,7 @@ public List getSlots() { return (List) children(); } - public int getIndexInSqlString() { + public Pair getIndexInSqlString() { return indexInSqlString; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java index 1d6c725db09d90..e8261f6391dda9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Lambda.java @@ -132,7 +132,7 @@ public String toSql() { if (argumentNames.size() > 1) { argStr = argumentNames.stream().collect(Collectors.joining(", ", "(", ")")); } - builder.append(String.format("%s -> %s", argStr, getLambdaFunction().toString())); + builder.append(String.format("%s -> %s", argStr, getLambdaFunction().toSql())); for (int i = 1; i < getArguments().size(); i++) { builder.append(", ").append(getArgument(i).toSql()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index 06115ee2778b52..fb3388b590ce3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -42,7 +42,6 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; @@ -63,6 +62,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; /** @@ -132,7 +132,7 @@ public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { CreateViewStmt createViewStmt = new CreateViewStmt(ifNotExists, viewName.transferToTableName(), cols, comment, null); // expand star(*) in project list - Map indexStringSqlMap = collectBoundStar(analyzedPlan); + Map, String> indexStringSqlMap = collectBoundStar(analyzedPlan); String rewrittenSql = rewriteStarToColumn(indexStringSqlMap); // rewrite project alias @@ -158,14 +158,14 @@ private Plan parseAndAnalyzeView(String ddlSql, ConnectContext ctx) { return viewContext.getRewritePlan(); } - private String rewriteStarToColumn(Map indexStringSqlMap) { + private String rewriteStarToColumn(Map, String> indexStringSqlMap) { StringBuilder builder = new StringBuilder(); int beg = 0; - for (Map.Entry entry : indexStringSqlMap.entrySet()) { - int index = entry.getKey(); - builder.append(querySql, beg, index); + for (Map.Entry, String> entry : indexStringSqlMap.entrySet()) { + Pair index = entry.getKey(); + builder.append(querySql, beg, index.first); builder.append(entry.getValue()); - beg = index + 1; + beg = index.second + 1; } builder.append(querySql, beg, querySql.length()); return builder.toString(); @@ -218,8 +218,8 @@ private void createFinalCols() throws org.apache.doris.common.AnalysisException } } - private Map collectBoundStar(Plan plan) { - Map result = Maps.newTreeMap(); + private Map, String> collectBoundStar(Plan plan) { + TreeMap, String> result = new TreeMap<>(new Pair.PairComparator<>()); plan.foreach(node -> { if (node instanceof LogicalProject) { LogicalProject project = (LogicalProject) node; From a3c470df0f487e6eb043d98a9409f3fd975d52d1 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sun, 24 Mar 2024 21:39:02 +0800 Subject: [PATCH 07/23] [feat](nereids) support create view in nereids --- regression-test/suites/ddl_p0/test_create_view.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/ddl_p0/test_create_view.groovy b/regression-test/suites/ddl_p0/test_create_view.groovy index 4b93c879101095..091aa4c679c1fe 100644 --- a/regression-test/suites/ddl_p0/test_create_view.groovy +++ b/regression-test/suites/ddl_p0/test_create_view.groovy @@ -43,6 +43,7 @@ suite("test_create_view") { "dynamic_partition.buckets" = "3" ); """ + sql "drop view if exists test_count_distinct" sql """ CREATE VIEW IF NOT EXISTS test_count_distinct ( From 4cf6fdd12434a11fe7260396c4e91da0449cb2bc Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 25 Mar 2024 11:30:25 +0800 Subject: [PATCH 08/23] [feat](nereids) support create view in nereids --- .../trees/expressions/WindowFrame.java | 27 +- .../plans/commands/info/CreateViewInfo.java | 3 +- .../data/ddl_p0/test_create_view_nereids.out | 140 ++++++++++ .../suites/ddl_p0/test_create_view.groovy | 3 +- .../ddl_p0/test_create_view_nereids.groovy | 255 ++++++++++++++++++ 5 files changed, 424 insertions(+), 4 deletions(-) create mode 100644 regression-test/data/ddl_p0/test_create_view_nereids.out create mode 100644 regression-test/suites/ddl_p0/test_create_view_nereids.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java index 90449ed0f86bad..1f9460ea0278b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java @@ -99,7 +99,7 @@ public String toSql() { StringBuilder sb = new StringBuilder(); sb.append(frameUnits + " "); if (rightBoundary != null) { - sb.append("BETWEEN " + leftBoundary + " AND " + rightBoundary); + sb.append("BETWEEN " + leftBoundary.toSql() + " AND " + rightBoundary.toSql()); } else { sb.append(leftBoundary); } @@ -215,6 +215,31 @@ public String toString() { return sb.toString(); } + public String toSql() { + StringBuilder sb = new StringBuilder(); + boundOffset.ifPresent(value -> sb.append(value + " ")); + switch (frameBoundType) { + case UNBOUNDED_PRECEDING: + sb.append("UNBOUNDED PRECEDING"); + break; + case UNBOUNDED_FOLLOWING: + sb.append("UNBOUNDED FOLLOWING"); + break; + case CURRENT_ROW: + sb.append("CURRENT ROW"); + break; + case PRECEDING: + sb.append("PRECEDING"); + break; + case FOLLOWING: + sb.append("FOLLOWING"); + break; + default: + break; + } + return sb.toString(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index fb3388b590ce3a..7abe361c8be384 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -188,7 +188,8 @@ private String rewriteProjectsToUserDefineAlias(String resSql) { for (int i = 0; i < projectExprs.size(); ++i) { replaceWithColsBuilder.append(projectExprs.get(i).toSql()); replaceWithColsBuilder.append(" AS `"); - replaceWithColsBuilder.append(finalCols.get(i).getName()); + String escapeBacktick = finalCols.get(i).getName().replace("`", "``"); + replaceWithColsBuilder.append(escapeBacktick); replaceWithColsBuilder.append('`'); if (i != projectExprs.size() - 1) { replaceWithColsBuilder.append(", "); diff --git a/regression-test/data/ddl_p0/test_create_view_nereids.out b/regression-test/data/ddl_p0/test_create_view_nereids.out new file mode 100644 index 00000000000000..b922be4ffcdd9d --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_view_nereids.out @@ -0,0 +1,140 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_view_1 -- +1 [1, 2, 3] +2 [10, -2, 8] +3 [-1, 20, 0] + +-- !test_view_2 -- +1 [1, 2, 3] [1, 1, 1] +2 [10, -2, 8] [1, 0, 1] +3 [-1, 20, 0] [0, 1, 0] + +-- !test_view_3 -- +1 [1, 2, 3] [1, 2, 3] [1, 2, 3] +2 [10, -2, 8] [10, 8] [10, 8] +3 [-1, 20, 0] [20] [20] + +-- !test_view_4 -- +1 [1, 2, 3] [1, 2, 3] [1, 2, 3] +2 [10, -2, 8] [10, 8] [10, 8] +3 [-1, 20, 0] [20] [20] + +-- !test_view_5 -- +1 [1, 2, 3] [1, 1, 1] +2 [10, -2, 8] [1, 0, 1] +3 [-1, 20, 0] [0, 1, 0] + +-- !test_view_6 -- +v1 CREATE VIEW `v1` COMMENT 'VIEW' AS SELECT\n `regression_test_ddl_p0`.`view_column_name_test_nereids`.`error_code` AS `error_code`, 1 AS `1`, 'string' AS `'string'`, now() AS `now()`, dayofyear(`regression_test_ddl_p0`.`view_column_name_test_nereids`.`op_time`) AS `dayofyear(``regression_test_ddl_p0``.``view_column_name_test_nereids``.``op_time``)`, cast(`regression_test_ddl_p0`.`view_column_name_test_nereids`.`source` as BIGINT) AS `cast(``regression_test_ddl_p0``.``view_column_name_test_nereids``.``source`` as BIGINT)`, min(`regression_test_ddl_p0`.`view_column_name_test_nereids`.`timestamp`) OVER(ORDER BY `regression_test_ddl_p0`.`view_column_name_test_nereids`.`op_time` desc ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) AS `min(``regression_test_ddl_p0``.``view_column_name_test_nereids``.``timestamp``) OVER(ORDER BY ``regression_test_ddl_p0``.``view_column_name_test_nereids``.``op_time`` desc ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)`, (1 > 2) AS `(1 > 2)`, (2 + 3) AS `(2 + 3)`, 1 IN (1, 2, 3, 4) AS `1 IN (1, 2, 3, 4)`, (`regression_test_ddl_p0`.`view_column_name_test_nereids`.`remark` like '%like') AS `(``regression_test_ddl_p0``.``view_column_name_test_nereids``.``remark`` like '%like')`, CASE WHEN (`regression_test_ddl_p0`.`view_column_name_test_nereids`.`remark` = 's') THEN 1 ELSE 2 END AS `CASE WHEN (``regression_test_ddl_p0``.``view_column_name_test_nereids``.``remark`` = 's') THEN 1 ELSE 2 END`, (cast(TRUE as TINYINT) | cast(FALSE as TINYINT)) AS `(cast(TRUE as TINYINT) | cast(FALSE as TINYINT))` \n FROM \n view_column_name_test_nereids; + +-- !test_with_as -- +1 1 2 +1 1 4 +1 3 6 +2 1 3 +2 1 4 +2 1 7 +2 3 5 +2 3 9 +2 4 2 +3 2 8 +3 5 \N +3 5 6 +3 5 6 +3 5 8 +4 5 6 +6 \N 6 +6 7 1 + +-- !test_with_as_sql -- +test_view_with_as CREATE VIEW `test_view_with_as` COMMENT 'VIEW' AS (\n with t1 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view), t2 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view), \n t3 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view) SELECT `t1`.`pk` AS `pk`, `t1`.`a` AS `a`, `t1`.`b` AS `b` FROM t1); utf8mb4 utf8mb4_0900_bin + +-- !test_union -- +1 1 2 +1 1 2 +1 1 4 +1 1 4 +1 3 6 +1 3 6 +2 1 3 +2 1 3 +2 1 4 +2 1 4 +2 1 7 +2 1 7 +2 3 5 +2 3 5 +2 3 9 +2 3 9 +2 4 2 +2 4 2 +3 2 8 +3 2 8 +3 5 \N +3 5 \N +3 5 6 +3 5 6 +3 5 6 +3 5 6 +3 5 8 +3 5 8 +4 5 6 +4 5 6 +6 \N 6 +6 \N 6 +6 7 1 +6 7 1 + +-- !test_union_sql -- +test_view_union CREATE VIEW `test_view_union` COMMENT 'VIEW' AS (\nselect `regression_test_ddl_p0`.`mal_test_view`.`pk` AS `c1`, `regression_test_ddl_p0`.`mal_test_view`.`a` AS `c2`, `regression_test_ddl_p0`.`mal_test_view`.`b` AS `c3` from mal_test_view Union all SELECT * FROM mal_test_view); utf8mb4 utf8mb4_0900_bin + +-- !test_count_star -- +17 + +-- !test_count_star_sql -- +test_view_count_star CREATE VIEW `test_view_count_star` COMMENT 'VIEW' AS (select count(*) AS `c1` from mal_test_view having count(*) > 0); utf8mb4 utf8mb4_0900_bin + +-- !test_expression -- +\N \N 6 +2 4 2 +2 4 3 +2 4 4 +2 4 4 +2 4 7 +3 5 8 +4 6 5 +4 6 6 +4 6 9 +5 7 2 +6 8 \N +6 8 6 +6 8 6 +6 8 6 +6 8 8 +8 10 1 + +-- !test_expression_sql -- +test_view_expression CREATE VIEW `test_view_expression` COMMENT 'VIEW' AS (select (`regression_test_ddl_p0`.`mal_test_view`.`a` + 1) AS `c1`, ((abs(`regression_test_ddl_p0`.`mal_test_view`.`a`) + 2) + 1) AS `c2`, substring(cast(`regression_test_ddl_p0`.`mal_test_view`.`b` as VARCHAR(10)), 1, 10) AS `c3` from mal_test_view); utf8mb4 utf8mb4_0900_bin + +-- !test_alias -- +\N \N 6 +2 4 2 +2 4 3 +2 4 4 +2 4 4 +2 4 7 +3 5 8 +4 6 5 +4 6 6 +4 6 9 +5 7 2 +6 8 \N +6 8 6 +6 8 6 +6 8 6 +6 8 8 +8 10 1 + +-- !test_alias_sql -- +test_view_alias CREATE VIEW `test_view_alias` COMMENT 'VIEW' AS (\n select `t`.`c8` AS `c1`, `t`.`c2` AS `c2`, `t`.`c1` AS `c3` from (select a+1 c8,abs(a)+2+1 as c2, cast(b as varchar(10)) as c1 from mal_test_view) t); utf8mb4 utf8mb4_0900_bin + diff --git a/regression-test/suites/ddl_p0/test_create_view.groovy b/regression-test/suites/ddl_p0/test_create_view.groovy index 091aa4c679c1fe..e66798e50bbb19 100644 --- a/regression-test/suites/ddl_p0/test_create_view.groovy +++ b/regression-test/suites/ddl_p0/test_create_view.groovy @@ -16,8 +16,7 @@ // under the License. suite("test_create_view") { - sql "SET enable_nereids_planner=true;" - sql "SET enable_fallback_to_original_planner=false;" + sql "SET enable_nereids_planner=false;" sql """DROP TABLE IF EXISTS count_distinct""" sql """ CREATE TABLE IF NOT EXISTS count_distinct diff --git a/regression-test/suites/ddl_p0/test_create_view_nereids.groovy b/regression-test/suites/ddl_p0/test_create_view_nereids.groovy new file mode 100644 index 00000000000000..aa631ef9e0d37a --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_view_nereids.groovy @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_create_view_nereids") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + sql """DROP TABLE IF EXISTS count_distinct_nereids""" + sql """ + CREATE TABLE IF NOT EXISTS count_distinct_nereids + ( + RQ DATE NOT NULL COMMENT "日期", + v1 VARCHAR(100) NOT NULL COMMENT "字段1", + v2 VARCHAR(100) NOT NULL COMMENT "字段2", + v3 VARCHAR(100) REPLACE_IF_NOT_NULL COMMENT "字段3" + ) + AGGREGATE KEY(RQ,v1,v2) + PARTITION BY RANGE(RQ) + ( + PARTITION p20220908 VALUES LESS THAN ('2022-09-09') + ) + DISTRIBUTED BY HASH(v1,v2) BUCKETS 3 + PROPERTIES( + "replication_num" = "1", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.start" = "-3", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "3" + ); + """ + sql """ + CREATE VIEW IF NOT EXISTS test_count_distinct_nereids + ( + RQ comment "日期", + v1 comment "v1", + v2 comment "v2", + v3 comment "v3" + ) + AS + select aa.RQ as RQ, aa.v1 as v1,aa.v2 as v2 , bb.v3 as v3 from + ( + select RQ, count(distinct v1) as v1 , count(distinct v2 ) as v2 + from count_distinct_nereids + group by RQ + ) aa + LEFT JOIN + ( + select RQ, max(v3) as v3 + from count_distinct_nereids + group by RQ + ) bb + on aa.RQ = bb.RQ; + """ + + sql """select * from test_count_distinct_nereids""" + sql """DROP VIEW IF EXISTS test_count_distinct_nereids""" + sql """DROP TABLE IF EXISTS count_distinct_nereids""" + + sql """DROP TABLE IF EXISTS test_view_t1""" + sql """ + CREATE TABLE `test_view_t1` ( + k1 int, + k2 date, + v1 int + ) ENGINE=OLAP + UNIQUE KEY(`k1`,`k2`) + COMMENT '测试' + PARTITION BY RANGE(k2) ( + PARTITION p1 VALUES [('2023-07-01'), ('2023-07-10')), + PARTITION p2 VALUES [('2023-07-11'), ('2023-07-20')) + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + sql """DROP TABLE IF EXISTS test_view_t2""" + sql """ + CREATE TABLE `test_view_t2` ( + k1 int, + k2 date, + v1 int + ) ENGINE=OLAP + UNIQUE KEY(`k1`,`k2`) + COMMENT '测试' + PARTITION BY RANGE(k2) ( + PARTITION p1 VALUES [('2023-07-01'), ('2023-07-05')), + PARTITION p2 VALUES [('2023-07-05'), ('2023-07-15')) + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); """ + sql """ + CREATE VIEW IF NOT EXISTS my_view_nereids AS + SELECT test_view_t1.* FROM test_view_t1 PARTITION(p1) JOIN test_view_t2 PARTITION(p2) ON test_view_t1.k1 = test_view_t2.k1; """ + sql """SELECT * FROM my_view_nereids""" + sql """DROP VIEW IF EXISTS my_view_nereids""" + sql """DROP TABLE IF EXISTS test_view_t1""" + sql """DROP TABLE IF EXISTS test_view_t2""" + + + sql """DROP TABLE IF EXISTS view_baseall_nereids""" + sql """DROP VIEW IF EXISTS test_view7_nereids""" + sql """DROP VIEW IF EXISTS test_view8""" + sql """ + CREATE TABLE `view_baseall_nereids` ( + `k1` int(11) NULL, + `k3` array NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 5 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "is_being_synced" = "false", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false" + ); + """ + sql """insert into view_baseall_nereids values(1,[1,2,3]);""" + sql """insert into view_baseall_nereids values(2,[10,-2,8]);""" + sql """insert into view_baseall_nereids values(3,[-1,20,0]);""" + + qt_test_view_1 """ select * from view_baseall_nereids order by k1; """ + qt_test_view_2 """ select *, array_map(x->x>0,k3) from view_baseall_nereids order by k1; """ + qt_test_view_3 """ select *, array_filter(x->x>0,k3),array_filter(`k3`, array_map(x -> x > 0, `k3`)) from view_baseall_nereids order by k1; """ + + + sql """ + create view IF NOT EXISTS test_view7_nereids (k1,k2,k3,k4) as + select *, array_filter(x->x>0,k3),array_filter(`k3`, array_map(x -> x > 0, `k3`)) from view_baseall_nereids order by k1; + """ + qt_test_view_4 """ select * from test_view7_nereids order by k1; """ + + sql """ + create view IF NOT EXISTS test_view8_nereids (k1,k2,k3) as + select *, array_map(x->x>0,k3) from view_baseall_nereids order by k1; + """ + qt_test_view_5 """ select * from test_view8_nereids order by k1; """ + + sql """DROP TABLE IF EXISTS view_column_name_test_nereids""" + sql """ + CREATE TABLE IF NOT EXISTS view_column_name_test_nereids + ( + `timestamp` DATE NOT NULL COMMENT "['0000-01-01', '9999-12-31']", + `type` TINYINT NOT NULL COMMENT "[-128, 127]", + `error_code` INT COMMENT "[-2147483648, 2147483647]", + `error_msg` VARCHAR(300) COMMENT "[1-65533]", + `op_id` BIGINT COMMENT "[-9223372036854775808, 9223372036854775807]", + `op_time` DATETIME COMMENT "['0000-01-01 00:00:00', '9999-12-31 23:59:59']", + `target` float COMMENT "4 字节", + `source` double COMMENT "8 字节", + `lost_cost` decimal(12,2) COMMENT "", + `remark` string COMMENT "1m size", + `op_userid` LARGEINT COMMENT "[-2^127 + 1 ~ 2^127 - 1]", + `plate` SMALLINT COMMENT "[-32768, 32767]", + `iscompleted` boolean COMMENT "true 或者 false" + ) + DISTRIBUTED BY HASH(`type`) BUCKETS 1 + PROPERTIES ('replication_num' = '1'); + """ + + sql """ + DROP VIEW IF EXISTS v1 + """ + sql """ + CREATE VIEW v1 AS + SELECT + error_code, + 1, + 'string', + now(), + dayofyear(op_time), + cast (source AS BIGINT), + min(`timestamp`) OVER ( + ORDER BY + op_time DESC ROWS BETWEEN UNBOUNDED PRECEDING + AND 1 FOLLOWING + ), + 1 > 2, + 2 + 3, + 1 IN (1, 2, 3, 4), + remark LIKE '%like', + CASE WHEN remark = 's' THEN 1 ELSE 2 END, + TRUE | FALSE + FROM + view_column_name_test_nereids + """ + qt_test_view_6 """ SHOW VIEW FROM view_column_name_test_nereids;""" + + // test with as + sql """ + DROP TABLE IF EXISTS mal_test_view + """ + + sql """ + create table mal_test_view(pk int, a int, b int) distributed by hash(pk) buckets 10 + properties('replication_num' = '1'); + """ + + sql """ + insert into mal_test_view values(2,1,3),(1,1,2),(3,5,6),(6,null,6),(4,5,6),(2,1,4),(2,3,5),(1,1,4) + ,(3,5,6),(3,5,null),(6,7,1),(2,1,7),(2,4,2),(2,3,9),(1,3,6),(3,5,8),(3,2,8); + """ + sql "DROP VIEW if exists test_view_with_as" + sql """CREATE VIEW if not exists test_view_with_as AS ( + with t1 as (select * from mal_test_view), t2 as (select * from mal_test_view), + t3 as (select * from mal_test_view) SELECT * FROM t1);""" + qt_test_with_as "select * from test_view_with_as order by pk, a, b" + qt_test_with_as_sql "show create view test_view_with_as" + + // test union + sql "DROP VIEW if exists test_view_union" + sql "CREATE VIEW test_view_union(c1,c2,c3) AS (\n" + + "select * from mal_test_view Union all SELECT * FROM mal_test_view);" + qt_test_union "select c1,c2,c3 from test_view_union order by c1,c2,c3" + qt_test_union_sql "show create view test_view_union" + + // test count(*) + sql "drop view if exists test_view_count_star;" + sql "CREATE VIEW test_view_count_star(c1) AS (select count(*) from mal_test_view having count(*) > 0);" + qt_test_count_star "select c1 from test_view_count_star order by c1" + qt_test_count_star_sql "show create view test_view_count_star" + + // test expression + sql "drop view if exists test_view_expression;" + sql """CREATE VIEW test_view_expression(c1,c2,c3) AS (select a+1,abs(a)+2+1 as c2, cast(b as varchar(10)) as c1 from mal_test_view);""" + qt_test_expression "select * from test_view_expression order by c1,c2,c3" + qt_test_expression_sql "show create view test_view_expression;" + + // test alias + sql "drop view if exists test_view_alias;" + sql """CREATE VIEW test_view_alias(c1,c2,c3) AS ( + select c8 as c9, c2 as c3, c1 as c4 from (select a+1 c8,abs(a)+2+1 as c2, cast(b as varchar(10)) as c1 from mal_test_view) t);""" + qt_test_alias "select * from test_view_alias order by c1,c2,c3;" + qt_test_alias_sql "show create view test_view_alias;" + +} From 4922b796e5cce4058f681e8b3c01a2d2523f6592 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 25 Mar 2024 11:35:53 +0800 Subject: [PATCH 09/23] [feat](nereids) support create view in nereids --- .../org/apache/doris/nereids/trees/expressions/WindowFrame.java | 1 + .../doris/nereids/trees/plans/commands/info/CreateViewInfo.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java index 1f9460ea0278b8..5cbb93ce3748ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/WindowFrame.java @@ -215,6 +215,7 @@ public String toString() { return sb.toString(); } + /** toSql*/ public String toSql() { StringBuilder sb = new StringBuilder(); boundOffset.ifPresent(value -> sb.append(value + " ")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index 7abe361c8be384..6e67b7a8697e9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -52,7 +52,6 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.tree.ParseTree; From 6bb25b1524ae9dbf706331e9c59861d839fbc75c Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 26 Mar 2024 15:02:05 +0800 Subject: [PATCH 10/23] [feat](nereids) support create view in nereids --- .../doris/nereids/jobs/executor/Analyzer.java | 4 +- .../nereids/properties/LogicalProperties.java | 12 +- .../properties/UnboundLogicalProperties.java | 2 +- .../nereids/rules/analysis/BindRelation.java | 2 +- .../nereids/trees/plans/AbstractPlan.java | 9 +- .../doris/nereids/trees/plans/FakePlan.java | 8 +- .../doris/nereids/trees/plans/Plan.java | 6 + .../plans/commands/info/CreateViewInfo.java | 130 +++++++++++++++--- .../trees/plans/logical/LogicalAggregate.java | 5 + .../trees/plans/logical/LogicalApply.java | 12 ++ .../plans/logical/LogicalAssertNumRows.java | 7 + .../trees/plans/logical/LogicalCTE.java | 6 + .../trees/plans/logical/LogicalCTEAnchor.java | 6 + .../plans/logical/LogicalCTEConsumer.java | 6 + .../plans/logical/LogicalCTEProducer.java | 6 + .../plans/logical/LogicalCatalogRelation.java | 8 ++ .../plans/logical/LogicalCheckPolicy.java | 5 + .../LogicalDeferMaterializeOlapScan.java | 9 ++ .../logical/LogicalDeferMaterializeTopN.java | 8 ++ .../plans/logical/LogicalEmptyRelation.java | 5 + .../trees/plans/logical/LogicalFilter.java | 6 + .../trees/plans/logical/LogicalGenerate.java | 9 ++ .../trees/plans/logical/LogicalHaving.java | 6 + .../plans/logical/LogicalInlineTable.java | 5 + .../trees/plans/logical/LogicalJoin.java | 10 ++ .../trees/plans/logical/LogicalLimit.java | 6 + .../trees/plans/logical/LogicalOlapScan.java | 6 + .../plans/logical/LogicalOneRowRelation.java | 5 + .../plans/logical/LogicalPartitionTopN.java | 6 + .../trees/plans/logical/LogicalProject.java | 5 + .../trees/plans/logical/LogicalRepeat.java | 5 + .../plans/logical/LogicalSelectHint.java | 6 + .../plans/logical/LogicalSetOperation.java | 5 + .../trees/plans/logical/LogicalSink.java | 5 + .../trees/plans/logical/LogicalSort.java | 6 + .../plans/logical/LogicalSubQueryAlias.java | 7 + .../plans/logical/LogicalTVFRelation.java | 7 + .../trees/plans/logical/LogicalTopN.java | 6 + .../trees/plans/logical/LogicalView.java | 6 + .../trees/plans/logical/LogicalWindow.java | 9 ++ .../apache/doris/nereids/util/JoinUtils.java | 19 +++ .../data/ddl_p0/test_create_view_nereids.out | 10 +- .../ddl_p0/test_create_view_nereids.groovy | 4 +- 43 files changed, 382 insertions(+), 33 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index ffff22e7fb97b0..9ad10a30aa29bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -120,9 +120,7 @@ private static List buildAnalyzeViewJobs(Optional> outputMapSupplier; protected final Supplier> outputExprIdSetSupplier; protected final Supplier fdSupplier; + protected final Supplier> outputExpressionSupplier; private Integer hashCode = null; public LogicalProperties(Supplier> outputSupplier, - Supplier fdSupplier) { - this(outputSupplier, fdSupplier, ImmutableList::of); + Supplier fdSupplier, Supplier> outputExpressionSupplier) { + this(outputSupplier, fdSupplier, outputExpressionSupplier, ImmutableList::of); } /** @@ -58,6 +59,7 @@ public LogicalProperties(Supplier> outputSupplier, */ public LogicalProperties(Supplier> outputSupplier, Supplier fdSupplier, + Supplier> outputExpressionSupplier, Supplier> nonUserVisibleOutputSupplier) { this.outputSupplier = Suppliers.memoize( Objects.requireNonNull(outputSupplier, "outputSupplier can not be null") @@ -80,6 +82,8 @@ public LogicalProperties(Supplier> outputSupplier, this.fdSupplier = Suppliers.memoize( Objects.requireNonNull(fdSupplier, "FunctionalDependencies can not be null") ); + this.outputExpressionSupplier = Suppliers.memoize(Objects.requireNonNull(outputExpressionSupplier, + "outputExpressionSupplier can not be null")); } public List getOutput() { @@ -106,6 +110,10 @@ public List getOutputExprIds() { return outputExprIdsSupplier.get(); } + public List getOutputExpression() { + return outputExpressionSupplier.get(); + } + @Override public String toString() { return "LogicalProperties{" diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java index 886b7aceb13dda..eaca9ef7769177 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java @@ -32,7 +32,7 @@ public class UnboundLogicalProperties extends LogicalProperties { public static final UnboundLogicalProperties INSTANCE = new UnboundLogicalProperties(); private UnboundLogicalProperties() { - super(ImmutableList::of, () -> FunctionalDependencies.EMPTY_FUNC_DEPS); + super(ImmutableList::of, () -> FunctionalDependencies.EMPTY_FUNC_DEPS, ImmutableList::of); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 3903ef7f52aacc..c3050e13fee0bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -251,7 +251,7 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio String inlineViewDef = view.getInlineViewDef(); Plan viewBody = parseAndAnalyzeView(inlineViewDef, cascadesContext); LogicalView logicalView = new LogicalView<>(view, viewBody); - return new LogicalSubQueryAlias<>(tableQualifier, addProjectForView(view, logicalView)); + return new LogicalSubQueryAlias<>(tableQualifier, logicalView); case HMS_EXTERNAL_TABLE: HMSExternalTable hmsTable = (HMSExternalTable) table; if (Config.enable_query_hive_views && hmsTable.isView()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index 4be6d35dc94692..780f536bdb69ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -155,6 +156,11 @@ public Set getOutputExprIdSet() { return getLogicalProperties().getOutputExprIdSet(); } + @Override + public List getOutputExpression() { + return getLogicalProperties().getOutputExpression(); + } + @Override public LogicalProperties getLogicalProperties() { // TODO: use bound()? @@ -181,7 +187,8 @@ public LogicalProperties computeLogicalProperties() { Supplier fdSupplier = () -> this instanceof LogicalPlan ? ((LogicalPlan) this).computeFuncDeps(outputSupplier) : FunctionalDependencies.EMPTY_FUNC_DEPS; - return new LogicalProperties(outputSupplier, fdSupplier); + Supplier> outputExpressionSupplier = Suppliers.memoize(this::computeOutputExpression); + return new LogicalProperties(outputSupplier, fdSupplier, outputExpressionSupplier); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java index ad186c680ce08c..7bc494c2b7a693 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.MutableState; @@ -78,7 +79,7 @@ public List getExpressions() { @Override public LogicalProperties getLogicalProperties() { - return new LogicalProperties(ArrayList::new, () -> FunctionalDependencies.EMPTY_FUNC_DEPS); + return new LogicalProperties(ArrayList::new, () -> FunctionalDependencies.EMPTY_FUNC_DEPS, ArrayList::new); } @Override @@ -91,6 +92,11 @@ public List getOutput() { return new ArrayList<>(); } + @Override + public List getOutputExpression() { + return new ArrayList<>(); + } + @Override public String treeString() { return "DUMMY"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index 1b237c72fdc207..0d3fa194e032f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -97,6 +97,8 @@ default boolean displayExtraPlanFirst() { */ List getOutput(); + List getOutputExpression(); + /** * Get output slot set of the plan. */ @@ -128,6 +130,10 @@ default List computeOutput() { throw new IllegalStateException("Not support compute output for " + getClass().getName()); } + default List computeOutputExpression() { + throw new IllegalStateException("Not support compute output expression for " + getClass().getName()); + } + /** * Get the input relation ids set of the plan. * @return The result is collected from all inputs relations diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index 6e67b7a8697e9b..545a03e236b88b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -30,13 +30,24 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.DorisParser; +import org.apache.doris.nereids.DorisParser.ExpressionContext; +import org.apache.doris.nereids.DorisParser.NamedExpressionContext; +import org.apache.doris.nereids.DorisParser.NamedExpressionSeqContext; import org.apache.doris.nereids.DorisParserBaseVisitor; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.jobs.executor.AbstractBatchJobExecutor; +import org.apache.doris.nereids.jobs.rewrite.RewriteJob; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.analysis.AnalyzeCTE; +import org.apache.doris.nereids.rules.analysis.BindExpression; +import org.apache.doris.nereids.rules.analysis.BindRelation; +import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; +import org.apache.doris.nereids.rules.analysis.CheckPolicy; +import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.BoundStar; import org.apache.doris.nereids.trees.expressions.Expression; @@ -44,10 +55,12 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.qe.ConnectContext; @@ -60,6 +73,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; @@ -108,11 +122,11 @@ public void validate(ConnectContext ctx) throws UserException { } analyzedPlan = parseAndAnalyzeView(querySql, ctx); - OutermostProjectFinder projectPlanFinder = new OutermostProjectFinder(); - AtomicReference> outermostProject = new AtomicReference<>(); - analyzedPlan.accept(projectPlanFinder, outermostProject); - outputs.addAll(outermostProject.get().getOutput()); - projects.addAll(outermostProject.get().getProjects()); + OutermostPlanFinder outermostPlanFinder = new OutermostPlanFinder(); + AtomicReference outermostPlan = new AtomicReference<>(); + analyzedPlan.accept(outermostPlanFinder, outermostPlan); + outputs.addAll(outermostPlan.get().getOutput()); + projects.addAll(outermostPlan.get().getOutputExpression()); createFinalCols(); Set colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (Column col : finalCols) { @@ -135,7 +149,7 @@ public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { String rewrittenSql = rewriteStarToColumn(indexStringSqlMap); // rewrite project alias - rewrittenSql = rewriteProjectsToUserDefineAlias(rewrittenSql); + rewrittenSql = rewriteProjectsToUserDefineAlias2(rewrittenSql); createViewStmt.setInlineViewDef(rewrittenSql); createViewStmt.setFinalColumns(finalCols); @@ -151,9 +165,8 @@ private Plan parseAndAnalyzeView(String ddlSql, ConnectContext ctx) { } CascadesContext viewContext = CascadesContext.initContext( stmtCtx, parsedViewPlan, PhysicalProperties.ANY); - viewContext.keepOrShowPlanProcess(false, () -> { - viewContext.newAnalyzer(true).analyze(); - }); + AnalyzerForCreateView analyzer = new AnalyzerForCreateView(viewContext); + analyzer.analyze(); return viewContext.getRewritePlan(); } @@ -199,6 +212,33 @@ private String rewriteProjectsToUserDefineAlias(String resSql) { finder.getIndex().second + 1); } + private String rewriteProjectsToUserDefineAlias2(String resSql) { + IndexFinder finder = new IndexFinder(); + ParserRuleContext tree = NereidsParser.toAst(resSql, DorisParser::singleStatement); + finder.visit(tree); + if (simpleColumnDefinitions.isEmpty()) { + return resSql; + } + List namedExpressionContexts = finder.getNamedExpressionContexts(); + StringBuilder replaceWithColsBuilder = new StringBuilder(); + for (int i = 0; i < namedExpressionContexts.size(); ++i) { + NamedExpressionContext namedExpressionContext = namedExpressionContexts.get(i); + int start = namedExpressionContext.expression().start.getStartIndex(); + int stop = namedExpressionContext.expression().stop.getStopIndex(); + replaceWithColsBuilder.append(resSql, start, stop + 1); + replaceWithColsBuilder.append(" AS `"); + String escapeBacktick = finalCols.get(i).getName().replace("`", "``"); + replaceWithColsBuilder.append(escapeBacktick); + replaceWithColsBuilder.append('`'); + if (i != namedExpressionContexts.size() - 1) { + replaceWithColsBuilder.append(", "); + } + } + String replaceWithCols = replaceWithColsBuilder.toString(); + return StringUtils.overlay(resSql, replaceWithCols, finder.getIndex().first, + finder.getIndex().second + 1); + } + private void createFinalCols() throws org.apache.doris.common.AnalysisException { if (simpleColumnDefinitions.isEmpty()) { for (Slot output : outputs) { @@ -231,22 +271,40 @@ private Map, String> collectBoundStar(Plan plan) { return result; } - private static class OutermostProjectFinder extends DefaultPlanVisitor>> { + private static class OutermostPlanFinder extends DefaultPlanVisitor> { boolean found = false; @Override - public Void visit(Plan plan, AtomicReference> target) { + public Void visit(Plan plan, AtomicReference target) { if (found) { return null; } - if (plan instanceof LogicalCTEProducer) { + target.set(plan); + found = true; + return null; + } + + @Override + public Void visitLogicalCTEAnchor(LogicalCTEAnchor cteAnchor, + AtomicReference target) { + if (found) { return null; - } else if (plan instanceof LogicalProject) { - target.set((LogicalProject) plan); - found = true; } - return super.visit(plan, target); + return super.visit(cteAnchor, target); + } + + @Override + public Void visitLogicalCTEProducer(LogicalCTEProducer cteProducer, + AtomicReference target) { + return null; + } + + @Override + public Void visitLogicalSetOperation(LogicalSetOperation setOperation, AtomicReference target) { + if (found) { + return null; + } + return super.visit(setOperation, target); } } @@ -255,6 +313,7 @@ private static class IndexFinder extends DorisParserBaseVisitor { private boolean found = false; private int startIndex; private int stopIndex; + private List namedExpressionContexts = Lists.newArrayList(); @Override public Void visitChildren(RuleNode node) { @@ -282,11 +341,48 @@ public Void visitSelectColumnClause(DorisParser.SelectColumnClauseContext ctx) { startIndex = ctx.getStart().getStartIndex(); stopIndex = ctx.getStop().getStopIndex(); found = true; + + // 这里默认*已经被替换了，不存在*或者* except + NamedExpressionSeqContext namedExpressionSeqContext = ctx.namedExpressionSeq(); + namedExpressionContexts = namedExpressionSeqContext.namedExpression(); return null; } public Pair getIndex() { return Pair.of(startIndex, stopIndex); } + + public List getNamedExpressionContexts() { + return namedExpressionContexts; + } + } + + private static class AnalyzerForCreateView extends AbstractBatchJobExecutor { + private final List jobs = buildAnalyzeViewJobs(); + + public AnalyzerForCreateView(CascadesContext cascadesContext) { + super(cascadesContext); + } + + public void analyze() { + execute(); + } + + @Override + public List getJobs() { + return jobs; + } + + private static List buildAnalyzeViewJobs() { + return jobs( + topDown(new AnalyzeCTE()), + topDown(new EliminateLogicalSelectHint()), + bottomUp( + new BindRelation(), + new CheckPolicy(), + new BindExpression() + ) + ); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 20647a3808ebe7..900add75f89e1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -390,4 +390,9 @@ public ImmutableSet computeFdItems(Supplier> outputSupplier) return builder.build(); } + + @Override + public List computeOutputExpression() { + return outputExpressions; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java index 517048c209aba4..2472247f1d1c64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InSubquery; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; @@ -161,6 +162,17 @@ public List computeOutput() { .build(); } + @Override + public List computeOutputExpression() { + return ImmutableList.builder() + .addAll(left().getOutputExpression()) + .addAll(markJoinSlotReference.isPresent() + ? ImmutableList.of(markJoinSlotReference.get()) : ImmutableList.of()) + .addAll(needAddSubOutputToProjects + ? ImmutableList.of(right().getOutputExpression().get(0)) : ImmutableList.of()) + .build(); + } + @Override public String toString() { return Utils.toSqlString("LogicalApply", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java index 0279c9701caaff..6abcba3ebb26d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.AssertNumRowsElement; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -118,4 +119,10 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr public List computeOutput() { return child().getOutput().stream().map(o -> o.withNullable(true)).collect(Collectors.toList()); } + + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java index 7a75dd5c1a4abf..8b6904ca8d69e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -70,6 +71,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + @Override public String toString() { return Utils.toSqlString("LogicalCTE", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java index 67aef52f979ead..321dadaa174037 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -83,6 +84,11 @@ public List computeOutput() { return right().getOutput(); } + @Override + public List computeOutputExpression() { + return right().getOutputExpression(); + } + public CTEId getCteId() { return cteId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java index 71b1c43f791191..2aa6bf4ee8555e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; @@ -145,6 +146,11 @@ public List computeOutput() { return ImmutableList.copyOf(producerToConsumerOutputMap.values()); } + @Override + public List computeOutputExpression() { + return ImmutableList.copyOf(producerToConsumerOutputMap.values()); + } + public CTEId getCteId() { return cteId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java index aeae13302d9245..8fab3a1d427ae8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -90,6 +91,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + @Override public String toString() { return Utils.toSqlString("LogicalCteProducer[" + id.asInt() + "]", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 4076e8348e2208..380d9a30c4d711 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -101,6 +101,14 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } + @Override + public List computeOutputExpression() { + return table.getBaseSchema() + .stream() + .map(col -> SlotReference.fromColumn(table, col, qualified(), this)) + .collect(ImmutableList.toImmutableList()); + } + public List getQualifier() { return qualifier; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java index bda3b1f49d570d..800a2a3ccd3a49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java @@ -80,6 +80,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + @Override public String toString() { return Utils.toSqlString("LogicalCheckPolicy"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java index a49e37d8df40a0..23b98e76d7aad6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; @@ -110,6 +111,14 @@ public List computeOutput() { .build(); } + @Override + public List computeOutputExpression() { + return ImmutableList.builder() + .addAll(logicalOlapScan.getOutput()) + .add(columnIdSlot) + .build(); + } + @Override public Plan withGroupExpression(Optional groupExpression) { return new LogicalDeferMaterializeOlapScan(logicalOlapScan, deferMaterializeSlotIds, columnIdSlot, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java index 5bf0afe1faef59..57b284aa1678ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; @@ -119,6 +120,13 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } + @Override + public List computeOutputExpression() { + return logicalTopN.getOutputExpression().stream() + .filter(s -> !(s.getExprId().equals(columnIdSlot.getExprId()))) + .collect(ImmutableList.toImmutableList()); + } + @Override public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java index ab816f7a40859d..9555ccc2fab9dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java @@ -100,6 +100,11 @@ public List getOutputs() { return projects; } + @Override + public List getOutputExpression() { + return projects; + } + @Override public Plan pruneOutputs(List prunedOutputs) { return withProjects(prunedOutputs); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index 4375124a22455d..907425d8e48c7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; @@ -87,6 +88,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + @Override public String toString() { return Utils.toSqlString("LogicalFilter[" + id.asInt() + "]", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java index 0fb3964e511f0a..2caa9e87d719f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.plans.Plan; @@ -130,6 +131,14 @@ public List computeOutput() { .build(); } + @Override + public List computeOutputExpression() { + return ImmutableList.builder() + .addAll(child().getOutput()) + .addAll(generatorOutput) + .build(); + } + @Override public String toString() { return Utils.toSqlString("LogicalGenerate", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java index e4d01f127482b5..dafa23406f2922 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -102,6 +103,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + @Override public int hashCode() { return Objects.hashCode(conjuncts); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java index b2a2a1d83ca3e7..009bbe89006871 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java @@ -82,6 +82,11 @@ public List computeOutput() { return ImmutableList.of(); } + @Override + public List computeOutputExpression() { + return ImmutableList.of(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index d18a454fe73296..07282352cf8b47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.DistributeType; import org.apache.doris.nereids.trees.plans.JoinType; @@ -264,6 +265,15 @@ public List computeOutput() { .build(); } + @Override + public List computeOutputExpression() { + return ImmutableList.builder() + .addAll(JoinUtils.getJoinOutput(joinType, left(), right())) + .addAll(isMarkJoin() + ? ImmutableList.of(markJoinSlotReference.get()) : ImmutableList.of()) + .build(); + } + @Override public String toString() { List args = Lists.newArrayList( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java index 482c7e26058f25..c56f7385ddf154 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.LimitPhase; @@ -90,6 +91,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + @Override public String toString() { return Utils.toSqlString("LogicalLimit", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index cb5d1847fef2a5..f5d4e6e4a7e2c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule; import org.apache.doris.nereids.trees.TableSample; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; @@ -348,6 +349,11 @@ public List computeOutput() { return slots.build(); } + @Override + public List computeOutputExpression() { + return computeOutput().stream().map(slot -> (NamedExpression) slot).collect(Collectors.toList()); + } + @Override public Set getInputRelations() { Set relationIdSet = Sets.newHashSet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index d338be5cc88185..0a77933f22ab74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -94,6 +94,11 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } + @Override + public List computeOutputExpression() { + return projects; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java index 69da96e8f7ebcb..9d246860b5d270 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.WindowExpression; @@ -111,6 +112,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + public WindowFuncType getFunction() { return function; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 4e19c5ca623efd..003121e5ed9645 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -275,4 +275,9 @@ public List getBoundStars() { public void setBoundStars(List boundStars) { this.boundStars = boundStars; } + + @Override + public List computeOutputExpression() { + return projects; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 5e0311afe1b518..ff738311182993 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -108,6 +108,11 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } + @Override + public List computeOutputExpression() { + return outputExpressions; + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalRepeat(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java index 127889ea7ed471..e974a75b00ce0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.SelectHint; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; @@ -105,6 +106,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + @Override public String toString() { String hintStr = this.hints.entrySet() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java index 2e4ddb55ff2f02..f8c3fa53305772 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -105,6 +105,11 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } + @Override + public List computeOutputExpression() { + return outputs; + } + public List> collectChildrenProjections() { return castCommonDataTypeOutputs(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java index ea2fa49e4664bb..3da49ac7f99c2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java @@ -66,6 +66,11 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } + @Override + public List computeOutputExpression() { + return outputExprs; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java index 9d9d321e659636..411372c20aac7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -66,6 +67,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + public List getOrderKeys() { return orderKeys; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index 68a451f1df1b19..16d41e30c09cdb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -42,6 +43,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * The node of logical plan for sub query and alias @@ -99,6 +101,11 @@ public List computeOutput() { return currentOutput.build(); } + @Override + public List computeOutputExpression() { + return computeOutput().stream().map(slot -> (NamedExpression) slot).collect(Collectors.toList()); + } + public String getAlias() { return qualifier.get(qualifier.size() - 1); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java index 8302c9345dcc17..3a728e69534aa8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; @@ -36,6 +37,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** LogicalTableValuedFunctionRelation */ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation, BlockFuncDepsPropagation { @@ -103,6 +105,11 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } + @Override + public List computeOutputExpression() { + return computeOutput().stream().map(slot -> (NamedExpression) slot).collect(Collectors.toList()); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalTVFRelation(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java index 63e3dc9c0b9717..f57f7e077970d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; @@ -72,6 +73,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + @Override public List getOrderKeys() { return orderKeys; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java index e8621b1754c757..43485788b03bfe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -128,6 +129,11 @@ public List computeOutput() { return child().getOutput(); } + @Override + public List computeOutputExpression() { + return child().getOutputExpression(); + } + @Override public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { return ((LogicalPlan) child()).computeFuncDeps(outputSupplier); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java index 457678787bd738..ac9d2294d3308a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java @@ -144,6 +144,15 @@ public List computeOutput() { .build(); } + @Override + public List computeOutputExpression() { + return new ImmutableList.Builder() + .addAll(child().getOutputExpression()) + .addAll(windowExpressions.stream() + .collect(ImmutableList.toImmutableList())) + .build(); + } + @Override public String toString() { return Utils.toSqlString("LogicalWindow", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index a73c183e4e75c1..29fa746e77f630 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapContains; @@ -388,6 +389,24 @@ public static List getJoinOutput(JoinType joinType, Plan left, Plan right) } } + public static List getJoinOutputExpression(JoinType joinType, Plan left, Plan right) { + switch (joinType) { + case LEFT_SEMI_JOIN: + case LEFT_ANTI_JOIN: + case NULL_AWARE_LEFT_ANTI_JOIN: + return ImmutableList.copyOf(left.getOutputExpression()); + case RIGHT_SEMI_JOIN: + case RIGHT_ANTI_JOIN: + return ImmutableList.copyOf(right.getOutputExpression()); + default: + return ImmutableList.builder() + .addAll(left.getOutputExpression()) + .addAll(right.getOutputExpression()) + .build(); + } + } + + public static boolean hasMarkConjuncts(Join join) { return !join.getMarkJoinConjuncts().isEmpty(); } diff --git a/regression-test/data/ddl_p0/test_create_view_nereids.out b/regression-test/data/ddl_p0/test_create_view_nereids.out index b922be4ffcdd9d..6de2ca95352a30 100644 --- a/regression-test/data/ddl_p0/test_create_view_nereids.out +++ b/regression-test/data/ddl_p0/test_create_view_nereids.out @@ -25,7 +25,7 @@ 3 [-1, 20, 0] [0, 1, 0] -- !test_view_6 -- -v1 CREATE VIEW `v1` COMMENT 'VIEW' AS SELECT\n `regression_test_ddl_p0`.`view_column_name_test_nereids`.`error_code` AS `error_code`, 1 AS `1`, 'string' AS `'string'`, now() AS `now()`, dayofyear(`regression_test_ddl_p0`.`view_column_name_test_nereids`.`op_time`) AS `dayofyear(``regression_test_ddl_p0``.``view_column_name_test_nereids``.``op_time``)`, cast(`regression_test_ddl_p0`.`view_column_name_test_nereids`.`source` as BIGINT) AS `cast(``regression_test_ddl_p0``.``view_column_name_test_nereids``.``source`` as BIGINT)`, min(`regression_test_ddl_p0`.`view_column_name_test_nereids`.`timestamp`) OVER(ORDER BY `regression_test_ddl_p0`.`view_column_name_test_nereids`.`op_time` desc ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) AS `min(``regression_test_ddl_p0``.``view_column_name_test_nereids``.``timestamp``) OVER(ORDER BY ``regression_test_ddl_p0``.``view_column_name_test_nereids``.``op_time`` desc ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING)`, (1 > 2) AS `(1 > 2)`, (2 + 3) AS `(2 + 3)`, 1 IN (1, 2, 3, 4) AS `1 IN (1, 2, 3, 4)`, (`regression_test_ddl_p0`.`view_column_name_test_nereids`.`remark` like '%like') AS `(``regression_test_ddl_p0``.``view_column_name_test_nereids``.``remark`` like '%like')`, CASE WHEN (`regression_test_ddl_p0`.`view_column_name_test_nereids`.`remark` = 's') THEN 1 ELSE 2 END AS `CASE WHEN (``regression_test_ddl_p0``.``view_column_name_test_nereids``.``remark`` = 's') THEN 1 ELSE 2 END`, (cast(TRUE as TINYINT) | cast(FALSE as TINYINT)) AS `(cast(TRUE as TINYINT) | cast(FALSE as TINYINT))` \n FROM \n view_column_name_test_nereids; +v1 CREATE VIEW `v1` COMMENT 'VIEW' AS SELECT\n error_code, \n 1, \n 'string', \n now(), \n dayofyear(op_time), \n cast (source AS BIGINT), \n min(`timestamp`) OVER (\n ORDER BY \n op_time DESC ROWS BETWEEN UNBOUNDED PRECEDING\n AND 1 FOLLOWING\n ), \n 1 > 2,\n 2 + 3,\n 1 IN (1, 2, 3, 4), \n remark LIKE '%like', \n CASE WHEN remark = 's' THEN 1 ELSE 2 END,\n TRUE | FALSE \n FROM \n view_column_name_test_nereids; -- !test_with_as -- 1 1 2 @@ -47,7 +47,7 @@ v1 CREATE VIEW `v1` COMMENT 'VIEW' AS SELECT\n `regression_test_ddl_p0`.`vi 6 7 1 -- !test_with_as_sql -- -test_view_with_as CREATE VIEW `test_view_with_as` COMMENT 'VIEW' AS (\n with t1 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view), t2 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view), \n t3 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view) SELECT `t1`.`pk` AS `pk`, `t1`.`a` AS `a`, `t1`.`b` AS `b` FROM t1); utf8mb4 utf8mb4_0900_bin +test_view_with_as CREATE VIEW `test_view_with_as` COMMENT 'VIEW' AS (\n with t1 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view), t2 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view), \n t3 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view) SELECT `t1`.`pk`, `t1`.`a`, `t1`.`b` FROM t1); utf8mb4 utf8mb4_0900_bin -- !test_union -- 1 1 2 @@ -86,7 +86,7 @@ test_view_with_as CREATE VIEW `test_view_with_as` COMMENT 'VIEW' AS (\n 6 7 1 -- !test_union_sql -- -test_view_union CREATE VIEW `test_view_union` COMMENT 'VIEW' AS (\nselect `regression_test_ddl_p0`.`mal_test_view`.`pk` AS `c1`, `regression_test_ddl_p0`.`mal_test_view`.`a` AS `c2`, `regression_test_ddl_p0`.`mal_test_view`.`b` AS `c3` from mal_test_view Union all SELECT * FROM mal_test_view); utf8mb4 utf8mb4_0900_bin +test_view_union CREATE VIEW `test_view_union` COMMENT 'VIEW' AS (\nselect `regression_test_ddl_p0`.`mal_test_view`.`pk` AS `c1`, `regression_test_ddl_p0`.`mal_test_view`.`a` AS `c2`, `regression_test_ddl_p0`.`mal_test_view`.`b` AS `c3` from mal_test_view Union all SELECT `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` FROM mal_test_view); utf8mb4 utf8mb4_0900_bin -- !test_count_star -- 17 @@ -114,7 +114,7 @@ test_view_count_star CREATE VIEW `test_view_count_star` COMMENT 'VIEW' AS (selec 8 10 1 -- !test_expression_sql -- -test_view_expression CREATE VIEW `test_view_expression` COMMENT 'VIEW' AS (select (`regression_test_ddl_p0`.`mal_test_view`.`a` + 1) AS `c1`, ((abs(`regression_test_ddl_p0`.`mal_test_view`.`a`) + 2) + 1) AS `c2`, substring(cast(`regression_test_ddl_p0`.`mal_test_view`.`b` as VARCHAR(10)), 1, 10) AS `c3` from mal_test_view); utf8mb4 utf8mb4_0900_bin +test_view_expression CREATE VIEW `test_view_expression` COMMENT 'VIEW' AS (select a+1 AS `c1`, abs(a)+2+1 AS `c2`, cast(b as varchar(10)) AS `c3` from mal_test_view); utf8mb4 utf8mb4_0900_bin -- !test_alias -- \N \N 6 @@ -136,5 +136,5 @@ test_view_expression CREATE VIEW `test_view_expression` COMMENT 'VIEW' AS (selec 8 10 1 -- !test_alias_sql -- -test_view_alias CREATE VIEW `test_view_alias` COMMENT 'VIEW' AS (\n select `t`.`c8` AS `c1`, `t`.`c2` AS `c2`, `t`.`c1` AS `c3` from (select a+1 c8,abs(a)+2+1 as c2, cast(b as varchar(10)) as c1 from mal_test_view) t); utf8mb4 utf8mb4_0900_bin +test_view_alias CREATE VIEW `test_view_alias` COMMENT 'VIEW' AS (\n select c8 AS `c1`, c2 AS `c2`, c1 AS `c3` from (select a+1 c8,abs(a)+2+1 as c2, cast(b as varchar(10)) as c1 from mal_test_view) t); utf8mb4 utf8mb4_0900_bin diff --git a/regression-test/suites/ddl_p0/test_create_view_nereids.groovy b/regression-test/suites/ddl_p0/test_create_view_nereids.groovy index aa631ef9e0d37a..e5a9c56ee4e2a8 100644 --- a/regression-test/suites/ddl_p0/test_create_view_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_view_nereids.groovy @@ -228,8 +228,8 @@ suite("test_create_view_nereids") { // test union sql "DROP VIEW if exists test_view_union" - sql "CREATE VIEW test_view_union(c1,c2,c3) AS (\n" + - "select * from mal_test_view Union all SELECT * FROM mal_test_view);" + sql """CREATE VIEW test_view_union(c1,c2,c3) AS + (select * from mal_test_view Union all SELECT * FROM mal_test_view);""" qt_test_union "select c1,c2,c3 from test_view_union order by c1,c2,c3" qt_test_union_sql "show create view test_view_union" From 3b9a8ace00b81478b67b566f0e11c15ceee0a272 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 29 Mar 2024 18:12:28 +0800 Subject: [PATCH 11/23] [feat](nereids) support create view in nereids --- .../apache/doris/nereids/properties/LogicalProperties.java | 5 +++++ .../doris/nereids/trees/expressions/SlotReference.java | 3 ++- .../nereids/trees/plans/commands/info/CreateViewInfo.java | 5 +---- .../main/java/org/apache/doris/nereids/util/JoinUtils.java | 2 +- regression-test/data/ddl_p0/test_create_view_nereids.out | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java index 0d97d6d21c4366..eec6318fcf1735 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java @@ -51,6 +51,11 @@ public LogicalProperties(Supplier> outputSupplier, this(outputSupplier, fdSupplier, outputExpressionSupplier, ImmutableList::of); } + public LogicalProperties(Supplier> outputSupplier, + Supplier fdSupplier) { + this(outputSupplier, fdSupplier, null, ImmutableList::of); + } + /** * constructor of LogicalProperties. * diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 989b4da08b4d72..ff8f51d6fc2834 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -188,7 +188,8 @@ public Optional getTable() { @Override public String toSql() { - return getQualifiedNameWithBacktick(); + // return getQualifiedNameWithBacktick(); + return name.get(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index 545a03e236b88b..f800f2b38a643a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -30,7 +30,6 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.DorisParser; -import org.apache.doris.nereids.DorisParser.ExpressionContext; import org.apache.doris.nereids.DorisParser.NamedExpressionContext; import org.apache.doris.nereids.DorisParser.NamedExpressionSeqContext; import org.apache.doris.nereids.DorisParserBaseVisitor; @@ -45,7 +44,6 @@ import org.apache.doris.nereids.rules.analysis.AnalyzeCTE; import org.apache.doris.nereids.rules.analysis.BindExpression; import org.apache.doris.nereids.rules.analysis.BindRelation; -import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; import org.apache.doris.nereids.rules.analysis.CheckPolicy; import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint; import org.apache.doris.nereids.trees.expressions.Alias; @@ -73,7 +71,6 @@ import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; @@ -264,7 +261,7 @@ private Map, String> collectBoundStar(Plan plan) { if (node instanceof LogicalProject) { LogicalProject project = (LogicalProject) node; for (BoundStar star : project.getBoundStars()) { - result.put(star.getIndexInSqlString(), star.toSql()); + result.put(star.getIndexInSqlString(), star.toSqlWithBacktick()); } } }); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index 29fa746e77f630..790855cb26efdd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -389,6 +389,7 @@ public static List getJoinOutput(JoinType joinType, Plan left, Plan right) } } + /** getJoinOutputExpression */ public static List getJoinOutputExpression(JoinType joinType, Plan left, Plan right) { switch (joinType) { case LEFT_SEMI_JOIN: @@ -406,7 +407,6 @@ public static List getJoinOutputExpression(JoinType joinType, P } } - public static boolean hasMarkConjuncts(Join join) { return !join.getMarkJoinConjuncts().isEmpty(); } diff --git a/regression-test/data/ddl_p0/test_create_view_nereids.out b/regression-test/data/ddl_p0/test_create_view_nereids.out index 6de2ca95352a30..82ebeaff7d679f 100644 --- a/regression-test/data/ddl_p0/test_create_view_nereids.out +++ b/regression-test/data/ddl_p0/test_create_view_nereids.out @@ -86,7 +86,7 @@ test_view_with_as CREATE VIEW `test_view_with_as` COMMENT 'VIEW' AS (\n 6 7 1 -- !test_union_sql -- -test_view_union CREATE VIEW `test_view_union` COMMENT 'VIEW' AS (\nselect `regression_test_ddl_p0`.`mal_test_view`.`pk` AS `c1`, `regression_test_ddl_p0`.`mal_test_view`.`a` AS `c2`, `regression_test_ddl_p0`.`mal_test_view`.`b` AS `c3` from mal_test_view Union all SELECT `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` FROM mal_test_view); utf8mb4 utf8mb4_0900_bin +test_view_union CREATE VIEW `test_view_union` COMMENT 'VIEW' AS (select `regression_test_ddl_p0`.`mal_test_view`.`pk` AS `c1`, `regression_test_ddl_p0`.`mal_test_view`.`a` AS `c2`, `regression_test_ddl_p0`.`mal_test_view`.`b` AS `c3` from mal_test_view Union all SELECT `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` FROM mal_test_view); utf8mb4 utf8mb4_0900_bin -- !test_count_star -- 17 From 2c66c186f41b0f4793a8f8ade743060f48067221 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sun, 31 Mar 2024 22:03:37 +0800 Subject: [PATCH 12/23] [feat](nereids) support create view in nereids --- .../nereids/analyzer/UnboundRelation.java | 39 ++++-- .../nereids/parser/LogicalPlanBuilder.java | 10 +- .../apache/doris/nereids/rules/RuleType.java | 1 + .../analysis/BindRelationForCreateView.java | 89 +++++++++++++ .../nereids/trees/expressions/BoundStar.java | 17 +++ .../plans/commands/info/CreateViewInfo.java | 56 +++++++- .../org/apache/doris/nereids/util/Utils.java | 8 ++ .../data/ddl_p0/test_create_view_nereids.out | 10 +- ...st_show_create_table_and_views_nereids.out | 46 +++++++ .../test_show_create_table_and_views.groovy | 1 + ...show_create_table_and_views_nereids.groovy | 126 ++++++++++++++++++ 11 files changed, 380 insertions(+), 23 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java create mode 100644 regression-test/data/show_p0/test_show_create_table_and_views_nereids.out create mode 100644 regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 4514ea05bfb9c9..485cdc6c1290c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.analyzer; import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; @@ -55,35 +56,44 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu private final Optional tableSample; private final Optional indexName; private TableScanParams scanParams; + private Pair indexInSqlString; + private List tableQualifier; public UnboundRelation(RelationId id, List nameParts) { this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of(), - ImmutableList.of(), Optional.empty(), Optional.empty(), null); + ImmutableList.of(), Optional.empty(), Optional.empty(), null, null); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart) { this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of(), - ImmutableList.of(), Optional.empty(), Optional.empty(), null); + ImmutableList.of(), Optional.empty(), Optional.empty(), null, null); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, null); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, null); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, TableScanParams scanParams) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, null); } public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample, indexName, null); + isTempPart, tabletIds, hints, tableSample, indexName, null, null); + } + + public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, + List tabletIds, List hints, Optional tableSample, Optional indexName, + TableScanParams scanParams, Pair indexInSqlString) { + this(id, nameParts, Optional.empty(), Optional.empty(), + partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString); } /** @@ -92,7 +102,7 @@ public UnboundRelation(RelationId id, List nameParts, Optional nameParts, Optional groupExpression, Optional logicalProperties, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, - TableScanParams scanParams) { + TableScanParams scanParams, Pair indexInSqlString) { super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties); this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null")); this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null")); @@ -102,6 +112,7 @@ public UnboundRelation(RelationId id, List nameParts, Optional getNameParts() { @@ -122,14 +133,14 @@ public LogicalProperties computeLogicalProperties() { public Plan withGroupExpression(Optional groupExpression) { return new UnboundRelation(relationId, nameParts, groupExpression, Optional.of(getLogicalProperties()), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, null); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new UnboundRelation(relationId, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample, indexName, null); + isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString); } @Override @@ -187,4 +198,16 @@ public Optional getTableSample() { public TableScanParams getScanParams() { return scanParams; } + + public Pair getIndexInSqlString() { + return indexInSqlString; + } + + public List getTableQualifier() { + return tableQualifier; + } + + public void setTableQualifier(List tableQualifier) { + this.tableQualifier = tableQualifier; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index abf95b1238331c..7d3a791bebe3db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1332,11 +1332,13 @@ public LogicalPlan visitTableName(TableNameContext ctx) { scanParams = new TableScanParams(ctx.optScanParams().funcName.getText(), map); } + MultipartIdentifierContext identifier = ctx.multipartIdentifier(); TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample()); LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy( new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, - Optional.ofNullable(tableSample), indexName, scanParams)); + Optional.ofNullable(tableSample), indexName, scanParams, + Pair.of(identifier.start.getStartIndex(), identifier.stop.getStopIndex()))); LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias()); for (LateralViewContext lateralViewContext : ctx.lateralView()) { plan = withGenerate(plan, lateralViewContext); @@ -2874,7 +2876,8 @@ private LogicalPlan withSelectQuerySpecification( if (!expressions.stream().allMatch(UnboundSlot.class::isInstance)) { throw new ParseException("only column name is supported in except clause", selectColumnCtx); } - project = new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of())), + project = new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of(), + Pair.of(selectColumnCtx.start.getStartIndex(), selectColumnCtx.stop.getStopIndex()))), expressions, isDistinct, aggregate); } else { List projects = getNamedExpressions(selectColumnCtx.namedExpressionSeq()); @@ -3054,7 +3057,8 @@ private LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContext if (!expressions.stream().allMatch(UnboundSlot.class::isInstance)) { throw new ParseException("only column name is supported in except clause", selectCtx); } - return new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of())), + return new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of(), + Pair.of(selectCtx.start.getStartIndex(), selectCtx.stop.getStopIndex()))), expressions, isDistinct, input); } else { List projects = getNamedExpressions(selectCtx.namedExpressionSeq()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index aa38af04ec50c4..db34bb32c7566a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -420,6 +420,7 @@ public enum RuleType { LOGICAL_GENERATE_TO_PHYSICAL_GENERATE(RuleTypeClass.IMPLEMENTATION), LOGICAL_WINDOW_TO_PHYSICAL_WINDOW_RULE(RuleTypeClass.IMPLEMENTATION), IMPLEMENTATION_SENTINEL(RuleTypeClass.IMPLEMENTATION), + BINDING_RELATION_FOR_CREATE_VIEW(RuleTypeClass.REWRITE), // sentinel, use to count rules SENTINEL(RuleTypeClass.SENTINEL), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java new file mode 100644 index 00000000000000..d14b25a44c0985 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java @@ -0,0 +1,89 @@ +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CTEContext; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Unbound; +import org.apache.doris.nereids.analyzer.UnboundRelation; +import org.apache.doris.nereids.hint.LeadingHint; +import org.apache.doris.nereids.pattern.MatchingContext; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Relation; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.util.RelationUtil; + +import java.util.List; +import java.util.Optional; + +public class BindRelationForCreateView extends OneAnalysisRuleFactory { + + // TODO: cte will be copied to a sub-query with different names but the id of the unbound relation in them + // are the same, so we use new relation id when binding relation, and will fix this bug later. + @Override + public Rule build() { + return unboundRelation().thenApply(ctx -> { + Plan plan = doBindRelation(ctx); + if (!(plan instanceof Unbound) && plan instanceof Relation) { + // init output and allocate slot id immediately, so that the slot id increase + // in the order in which the table appears. + LogicalProperties logicalProperties = plan.getLogicalProperties(); + logicalProperties.getOutput(); + } + return plan; + }).toRule(RuleType.BINDING_RELATION_FOR_CREATE_VIEW); + } + + private Plan doBindRelation(MatchingContext ctx) { + List nameParts = ctx.root.getNameParts(); + switch (nameParts.size()) { + case 1: { // table + // Use current database name from catalog. + return bindWithCurrentDb(ctx.cascadesContext, ctx.root); + } + case 2: + // db.table + // Use database name from table name parts. + case 3: { + // catalog.db.table + // Use catalog and database name from name parts. + return bind(ctx.cascadesContext, ctx.root); + } + default: + throw new IllegalStateException("Table name [" + ctx.root.getTableName() + "] is invalid."); + } + } + + private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRelation unboundRelation) { + String tableName = unboundRelation.getNameParts().get(0); + // check if it is a CTE's name + CTEContext cteContext = cascadesContext.getCteContext().findCTEContext(tableName).orElse(null); + if (cteContext != null) { + Optional analyzedCte = cteContext.getAnalyzedCTEPlan(tableName); + if (analyzedCte.isPresent()) { + LogicalCTEConsumer consumer = new LogicalCTEConsumer(unboundRelation.getRelationId(), + cteContext.getCteId(), tableName, analyzedCte.get()); + if (cascadesContext.isLeadingJoin()) { + LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading"); + leading.putRelationIdAndTableName(Pair.of(consumer.getRelationId(), tableName)); + leading.getRelationIdToScanMap().put(consumer.getRelationId(), consumer); + } + return consumer; + } + } + List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), + unboundRelation.getNameParts()); + unboundRelation.setTableQualifier(tableQualifier); + return unboundRelation; + } + + private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboundRelation) { + List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), + unboundRelation.getNameParts()); + unboundRelation.setTableQualifier(tableQualifier); + return unboundRelation; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java index 6dec2f5debcf39..f8a96fe85cebaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java @@ -22,9 +22,12 @@ import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import com.clearspring.analytics.util.Lists; import com.google.common.base.Preconditions; +import org.glassfish.jersey.internal.guava.Sets; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** BoundStar is used to wrap list of slots for temporary. */ @@ -55,6 +58,20 @@ public String toSqlWithBacktick() { .collect(Collectors.joining(", ")); } + /** toSqlWithBacktick */ + public String toSqlWithBacktick(List excepts) { + List outputs = Lists.newArrayList(); + Set exceptSet = Sets.newHashSet(); + exceptSet.addAll(excepts); + for (Expression child : children) { + if (!exceptSet.contains(child)) { + outputs.add(child); + } + } + return outputs.stream().map(slot -> ((SlotReference) slot).getQualifiedNameWithBacktick()) + .collect(Collectors.joining(", ")); + } + @SuppressWarnings({"unchecked", "rawtypes"}) public List getSlots() { return (List) children(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index f800f2b38a643a..efdb0c97a8fe08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.DorisParserBaseVisitor; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.jobs.executor.AbstractBatchJobExecutor; @@ -44,6 +45,7 @@ import org.apache.doris.nereids.rules.analysis.AnalyzeCTE; import org.apache.doris.nereids.rules.analysis.BindExpression; import org.apache.doris.nereids.rules.analysis.BindRelation; +import org.apache.doris.nereids.rules.analysis.BindRelationForCreateView; import org.apache.doris.nereids.rules.analysis.CheckPolicy; import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint; import org.apache.doris.nereids.trees.expressions.Alias; @@ -60,6 +62,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; +import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Lists; @@ -89,6 +92,7 @@ public class CreateViewInfo { private final List outputs = Lists.newArrayList(); private final List projects = Lists.newArrayList(); private Plan analyzedPlan; + private Map, String> relationMap; /** constructor*/ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String comment, LogicalPlan logicalQuery, @@ -143,6 +147,7 @@ public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { null); // expand star(*) in project list Map, String> indexStringSqlMap = collectBoundStar(analyzedPlan); + indexStringSqlMap.putAll(relationMap); String rewrittenSql = rewriteStarToColumn(indexStringSqlMap); // rewrite project alias @@ -162,9 +167,15 @@ private Plan parseAndAnalyzeView(String ddlSql, ConnectContext ctx) { } CascadesContext viewContext = CascadesContext.initContext( stmtCtx, parsedViewPlan, PhysicalProperties.ANY); - AnalyzerForCreateView analyzer = new AnalyzerForCreateView(viewContext); + AnalyzerForCreateView analyzer = new AnalyzerForCreateView(viewContext, true); analyzer.analyze(); - return viewContext.getRewritePlan(); + Plan planForRelation = viewContext.getRewritePlan(); + relationMap = collectUnBoundRelation(planForRelation); + CascadesContext viewContextForStar = CascadesContext.initContext( + stmtCtx, planForRelation, PhysicalProperties.ANY); + AnalyzerForCreateView analyzerForStar = new AnalyzerForCreateView(viewContextForStar, false); + analyzerForStar.analyze(); + return viewContextForStar.getRewritePlan(); } private String rewriteStarToColumn(Map, String> indexStringSqlMap) { @@ -255,13 +266,31 @@ private void createFinalCols() throws org.apache.doris.common.AnalysisException } } + private Map, String> collectUnBoundRelation(Plan plan) { + TreeMap, String> result = new TreeMap<>(new Pair.PairComparator<>()); + plan.foreach(node -> { + if (node instanceof UnboundRelation) { + UnboundRelation unboundRelation = (UnboundRelation) node; + List tableQualifier = unboundRelation.getTableQualifier(); + result.put(unboundRelation.getIndexInSqlString(), Utils.qualifiedNameWithBacktick(tableQualifier)); + } + }); + return result; + } + private Map, String> collectBoundStar(Plan plan) { TreeMap, String> result = new TreeMap<>(new Pair.PairComparator<>()); plan.foreach(node -> { if (node instanceof LogicalProject) { LogicalProject project = (LogicalProject) node; - for (BoundStar star : project.getBoundStars()) { - result.put(star.getIndexInSqlString(), star.toSqlWithBacktick()); + if (project.getExcepts().isEmpty()) { + for (BoundStar star : project.getBoundStars()) { + result.put(star.getIndexInSqlString(), star.toSqlWithBacktick()); + } + } else { + List namedExpressions = project.getExcepts(); + BoundStar star = project.getBoundStars().get(0); + result.put(star.getIndexInSqlString(), star.toSqlWithBacktick(namedExpressions)); } } }); @@ -355,10 +384,15 @@ public List getNamedExpressionContexts() { } private static class AnalyzerForCreateView extends AbstractBatchJobExecutor { - private final List jobs = buildAnalyzeViewJobs(); + private final List jobs; - public AnalyzerForCreateView(CascadesContext cascadesContext) { + public AnalyzerForCreateView(CascadesContext cascadesContext, boolean forRelation) { super(cascadesContext); + if (forRelation) { + jobs = buildAnalyzeViewJobsForRelation(); + } else { + jobs = buildAnalyzeViewJobsForStar(); + } } public void analyze() { @@ -370,10 +404,18 @@ public List getJobs() { return jobs; } - private static List buildAnalyzeViewJobs() { + private static List buildAnalyzeViewJobsForRelation() { return jobs( topDown(new AnalyzeCTE()), topDown(new EliminateLogicalSelectHint()), + bottomUp( + new BindRelationForCreateView() + ) + ); + } + + private static List buildAnalyzeViewJobsForStar() { + return jobs( bottomUp( new BindRelation(), new CheckPolicy(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java index 5bf976d904ecff..6c9636a1e43303 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java @@ -135,6 +135,14 @@ public static String qualifiedNameWithBacktick(List qualifiers, String n return StringUtils.join(qualifiedNameParts(qualifierWithBacktick, nameWithBacktick), "."); } + public static String qualifiedNameWithBacktick(List qualifiers) { + List qualifierWithBacktick = Lists.newArrayListWithCapacity(qualifiers.size()); + for (String qualifier : qualifiers) { + qualifierWithBacktick.add('`' + qualifier + '`'); + } + return StringUtils.join(qualifierWithBacktick, "."); + } + /** * Get sql string for plan. * diff --git a/regression-test/data/ddl_p0/test_create_view_nereids.out b/regression-test/data/ddl_p0/test_create_view_nereids.out index 82ebeaff7d679f..09a573593d6d37 100644 --- a/regression-test/data/ddl_p0/test_create_view_nereids.out +++ b/regression-test/data/ddl_p0/test_create_view_nereids.out @@ -25,7 +25,7 @@ 3 [-1, 20, 0] [0, 1, 0] -- !test_view_6 -- -v1 CREATE VIEW `v1` COMMENT 'VIEW' AS SELECT\n error_code, \n 1, \n 'string', \n now(), \n dayofyear(op_time), \n cast (source AS BIGINT), \n min(`timestamp`) OVER (\n ORDER BY \n op_time DESC ROWS BETWEEN UNBOUNDED PRECEDING\n AND 1 FOLLOWING\n ), \n 1 > 2,\n 2 + 3,\n 1 IN (1, 2, 3, 4), \n remark LIKE '%like', \n CASE WHEN remark = 's' THEN 1 ELSE 2 END,\n TRUE | FALSE \n FROM \n view_column_name_test_nereids; +v1 CREATE VIEW `v1` COMMENT 'VIEW' AS SELECT\n error_code, \n 1, \n 'string', \n now(), \n dayofyear(op_time), \n cast (source AS BIGINT), \n min(`timestamp`) OVER (\n ORDER BY \n op_time DESC ROWS BETWEEN UNBOUNDED PRECEDING\n AND 1 FOLLOWING\n ), \n 1 > 2,\n 2 + 3,\n 1 IN (1, 2, 3, 4), \n remark LIKE '%like', \n CASE WHEN remark = 's' THEN 1 ELSE 2 END,\n TRUE | FALSE \n FROM \n `internal`.`regression_test_ddl_p0`.`view_column_name_test_nereids`; -- !test_with_as -- 1 1 2 @@ -86,13 +86,13 @@ test_view_with_as CREATE VIEW `test_view_with_as` COMMENT 'VIEW' AS (\n 6 7 1 -- !test_union_sql -- -test_view_union CREATE VIEW `test_view_union` COMMENT 'VIEW' AS (select `regression_test_ddl_p0`.`mal_test_view`.`pk` AS `c1`, `regression_test_ddl_p0`.`mal_test_view`.`a` AS `c2`, `regression_test_ddl_p0`.`mal_test_view`.`b` AS `c3` from mal_test_view Union all SELECT `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` FROM mal_test_view); utf8mb4 utf8mb4_0900_bin +test_view_union CREATE VIEW `test_view_union` COMMENT 'VIEW' AS (select `regression_test_ddl_p0`.`mal_test_view`.`pk` AS `c1`, `regression_test_ddl_p0`.`mal_test_view`.`a` AS `c2`, `regression_test_ddl_p0`.`mal_test_view`.`b` AS `c3` from `internal`.`regression_test_ddl_p0`.`mal_test_view` Union all SELECT `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` FROM `internal`.`regression_test_ddl_p0`.`mal_test_view`); utf8mb4 utf8mb4_0900_bin -- !test_count_star -- 17 -- !test_count_star_sql -- -test_view_count_star CREATE VIEW `test_view_count_star` COMMENT 'VIEW' AS (select count(*) AS `c1` from mal_test_view having count(*) > 0); utf8mb4 utf8mb4_0900_bin +test_view_count_star CREATE VIEW `test_view_count_star` COMMENT 'VIEW' AS (select count(*) AS `c1` from `internal`.`regression_test_ddl_p0`.`mal_test_view` having count(*) > 0); utf8mb4 utf8mb4_0900_bin -- !test_expression -- \N \N 6 @@ -114,7 +114,7 @@ test_view_count_star CREATE VIEW `test_view_count_star` COMMENT 'VIEW' AS (selec 8 10 1 -- !test_expression_sql -- -test_view_expression CREATE VIEW `test_view_expression` COMMENT 'VIEW' AS (select a+1 AS `c1`, abs(a)+2+1 AS `c2`, cast(b as varchar(10)) AS `c3` from mal_test_view); utf8mb4 utf8mb4_0900_bin +test_view_expression CREATE VIEW `test_view_expression` COMMENT 'VIEW' AS (select a+1 AS `c1`, abs(a)+2+1 AS `c2`, cast(b as varchar(10)) AS `c3` from `internal`.`regression_test_ddl_p0`.`mal_test_view`); utf8mb4 utf8mb4_0900_bin -- !test_alias -- \N \N 6 @@ -136,5 +136,5 @@ test_view_expression CREATE VIEW `test_view_expression` COMMENT 'VIEW' AS (selec 8 10 1 -- !test_alias_sql -- -test_view_alias CREATE VIEW `test_view_alias` COMMENT 'VIEW' AS (\n select c8 AS `c1`, c2 AS `c2`, c1 AS `c3` from (select a+1 c8,abs(a)+2+1 as c2, cast(b as varchar(10)) as c1 from mal_test_view) t); utf8mb4 utf8mb4_0900_bin +test_view_alias CREATE VIEW `test_view_alias` COMMENT 'VIEW' AS (\n select c8 AS `c1`, c2 AS `c2`, c1 AS `c3` from (select a+1 c8,abs(a)+2+1 as c2, cast(b as varchar(10)) as c1 from `internal`.`regression_test_ddl_p0`.`mal_test_view`) t); utf8mb4 utf8mb4_0900_bin diff --git a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out new file mode 100644 index 00000000000000..d88325d3711d83 --- /dev/null +++ b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out @@ -0,0 +1,46 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !show -- +show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !select -- +1 1 30 +1 2 5 +1 3 10 +1 300 2 +2 1 100 +2 2 10 +2 3 44 +2 200 1111 +3 1 10 +3 2 1 +23 900 1 +100 100 1 +200 20 1 +300 20 1 + +-- !select -- +1 5 +2 10 +3 1 + +-- !show -- +show_create_table_and_views_view CREATE VIEW `show_create_table_and_views_view` COMMENT 'VIEW' AS SELECT user_id AS `user_id`, cost AS `cost` FROM `internal`.`show_create_table_and_views_db`.`show_create_table_and_views_table`\n WHERE good_id = 2; utf8mb4 utf8mb4_0900_bin + +-- !select -- +1 47 +2 1265 +3 11 +23 1 +100 1 +200 1 +300 1 + +-- !show -- +show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !show -- +show_create_table_and_views_like CREATE TABLE `show_create_table_and_views_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !show -- +show_create_table_and_views_like_with_rollup CREATE TABLE `show_create_table_and_views_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + diff --git a/regression-test/suites/show_p0/test_show_create_table_and_views.groovy b/regression-test/suites/show_p0/test_show_create_table_and_views.groovy index 6c6322e5c4aa03..2814ba8cd8ac29 100644 --- a/regression-test/suites/show_p0/test_show_create_table_and_views.groovy +++ b/regression-test/suites/show_p0/test_show_create_table_and_views.groovy @@ -16,6 +16,7 @@ // under the License. suite("test_show_create_table_and_views", "show") { + sql "SET enable_nereids_planner=false;" def ret = sql "SHOW FRONTEND CONFIG like '%enable_feature_binlog%';" logger.info("${ret}") if (ret.size() != 0 && ret[0].size() > 1 && ret[0][1] == 'false') { diff --git a/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy new file mode 100644 index 00000000000000..aa235b2b188afd --- /dev/null +++ b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_show_create_table_and_views_nereids", "show") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + def ret = sql "SHOW FRONTEND CONFIG like '%enable_feature_binlog%';" +// logger.info("${ret}") +// if (ret.size() != 0 && ret[0].size() > 1 && ret[0][1] == 'false') { +// logger.info("enable_feature_binlog=false in frontend config, no need to run this case.") +// return +// } + + String suiteName = "show_create_table_and_views" + String dbName = "${suiteName}_db" + String tableName = "${suiteName}_table" + String viewName = "${suiteName}_view" + String rollupName = "${suiteName}_rollup" + String likeName = "${suiteName}_like" + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `user_id` LARGEINT NOT NULL, + `good_id` LARGEINT NOT NULL, + `cost` BIGINT SUM DEFAULT "0" + ) + AGGREGATE KEY(`user_id`, `good_id`) + PARTITION BY RANGE(`good_id`) + ( + PARTITION p1 VALUES LESS THAN ("100"), + PARTITION p2 VALUES LESS THAN ("200"), + PARTITION p3 VALUES LESS THAN ("300"), + PARTITION p4 VALUES LESS THAN ("400"), + PARTITION p5 VALUES LESS THAN ("500"), + PARTITION p6 VALUES LESS THAN ("600"), + PARTITION p7 VALUES LESS THAN MAXVALUE + ) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """INSERT INTO ${dbName}.${tableName} VALUES + (1, 1, 10), + (1, 1, 20), + (1, 2, 5), + (1, 3, 10), + (2, 1, 0), + (2, 1, 100), + (3, 1, 10), + (2, 2, 10), + (2, 3, 44), + (3, 2, 1), + (100, 100, 1), + (200, 20, 1), + (300, 20, 1), + (1, 300, 2), + (2, 200, 1111), + (23, 900, 1)""" + + qt_show "SHOW CREATE TABLE ${dbName}.${tableName}" + qt_select "SELECT * FROM ${dbName}.${tableName} ORDER BY user_id, good_id" + + // create view and show + sql """ + CREATE VIEW IF NOT EXISTS ${dbName}.${viewName} (user_id, cost) + AS + SELECT user_id, cost FROM ${dbName}.${tableName} + WHERE good_id = 2 + """ + qt_select "SELECT * FROM ${dbName}.${viewName} ORDER BY user_id" + qt_show "SHOW CREATE VIEW ${dbName}.${viewName}" + + // create rollup + sql """ALTER TABLE ${dbName}.${tableName} + ADD ROLLUP ${rollupName} (user_id, cost) + """ + + def isAlterTableFinish = { -> + def records = sql """SHOW ALTER TABLE ROLLUP FROM ${dbName}""" + for (def row in records) { + if (row[5] == "${rollupName}" && row[8] == "FINISHED") { + return true + } + } + false + } + while (!isAlterTableFinish()) { + Thread.sleep(100) + } + + qt_select "SELECT user_id, SUM(cost) FROM ${dbName}.${tableName} GROUP BY user_id ORDER BY user_id" + qt_show "SHOW CREATE TABLE ${dbName}.${tableName}" + + // create like + sql "CREATE TABLE ${dbName}.${likeName} LIKE ${dbName}.${tableName}" + qt_show "SHOW CREATE TABLE ${dbName}.${likeName}" + + // create like with rollup + sql "CREATE TABLE ${dbName}.${likeName}_with_rollup LIKE ${dbName}.${tableName} WITH ROLLUP" + qt_show "SHOW CREATE TABLE ${dbName}.${likeName}_with_rollup" + + sql "DROP TABLE IF EXISTS ${dbName}.${likeName}_with_rollup FORCE" + sql "DROP TABLE ${dbName}.${likeName} FORCE" + sql "DROP VIEW ${dbName}.${viewName}" + sql "DROP TABLE ${dbName}.${tableName} FORCE" + sql "DROP DATABASE ${dbName} FORCE" +} + From 4a941dda85c42f4670d75e9ff6ffbc5e401eb4e4 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sun, 31 Mar 2024 22:35:34 +0800 Subject: [PATCH 13/23] [feat](nereids) support create view in nereids --- .../analysis/BindRelationForCreateView.java | 18 ++++++++++++++++++ .../nereids/trees/expressions/BoundStar.java | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java index d14b25a44c0985..3cb5fdc29f09cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package org.apache.doris.nereids.rules.analysis; import org.apache.doris.common.Pair; @@ -19,6 +36,7 @@ import java.util.List; import java.util.Optional; +/** BindRelationForCreateView */ public class BindRelationForCreateView extends OneAnalysisRuleFactory { // TODO: cte will be copied to a sub-query with different names but the id of the unbound relation in them diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java index f8a96fe85cebaf..5182894348826f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java @@ -22,9 +22,9 @@ import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import com.clearspring.analytics.util.Lists; import com.google.common.base.Preconditions; -import org.glassfish.jersey.internal.guava.Sets; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import java.util.List; import java.util.Set; From 2e40f0feece2491a85d6eb17b22b1036e70c4584 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 1 Apr 2024 14:12:18 +0800 Subject: [PATCH 14/23] [feat](nereids) support create view in nereids --- .../nereids/properties/LogicalProperties.java | 15 +-- .../nereids/trees/plans/AbstractPlan.java | 9 +- .../doris/nereids/trees/plans/FakePlan.java | 6 - .../doris/nereids/trees/plans/Plan.java | 2 - .../plans/commands/CreateViewCommand.java | 1 + .../plans/commands/info/CreateViewInfo.java | 82 +++--------- .../trees/plans/logical/LogicalAggregate.java | 5 - .../trees/plans/logical/LogicalApply.java | 12 -- .../plans/logical/LogicalAssertNumRows.java | 7 - .../trees/plans/logical/LogicalCTE.java | 6 - .../trees/plans/logical/LogicalCTEAnchor.java | 6 - .../plans/logical/LogicalCTEConsumer.java | 6 - .../plans/logical/LogicalCTEProducer.java | 6 - .../plans/logical/LogicalCatalogRelation.java | 8 -- .../plans/logical/LogicalCheckPolicy.java | 5 - .../LogicalDeferMaterializeOlapScan.java | 9 -- .../logical/LogicalDeferMaterializeTopN.java | 8 -- .../plans/logical/LogicalEmptyRelation.java | 5 - .../trees/plans/logical/LogicalFilter.java | 6 - .../trees/plans/logical/LogicalGenerate.java | 9 -- .../trees/plans/logical/LogicalHaving.java | 6 - .../plans/logical/LogicalInlineTable.java | 5 - .../trees/plans/logical/LogicalJoin.java | 10 -- .../trees/plans/logical/LogicalLimit.java | 6 - .../trees/plans/logical/LogicalOlapScan.java | 6 - .../plans/logical/LogicalOneRowRelation.java | 5 - .../plans/logical/LogicalPartitionTopN.java | 6 - .../trees/plans/logical/LogicalProject.java | 5 - .../trees/plans/logical/LogicalRepeat.java | 5 - .../plans/logical/LogicalSelectHint.java | 6 - .../plans/logical/LogicalSetOperation.java | 5 - .../trees/plans/logical/LogicalSink.java | 5 - .../trees/plans/logical/LogicalSort.java | 6 - .../plans/logical/LogicalSubQueryAlias.java | 7 - .../plans/logical/LogicalTVFRelation.java | 7 - .../trees/plans/logical/LogicalTopN.java | 6 - .../trees/plans/logical/LogicalView.java | 6 - .../trees/plans/logical/LogicalWindow.java | 9 -- .../apache/doris/nereids/util/JoinUtils.java | 19 --- ...st_show_create_table_and_views_nereids.out | 46 ------- .../test_show_create_table_and_views.groovy | 1 - ...show_create_table_and_views_nereids.groovy | 126 ------------------ 42 files changed, 23 insertions(+), 493 deletions(-) delete mode 100644 regression-test/data/show_p0/test_show_create_table_and_views_nereids.out delete mode 100644 regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java index eec6318fcf1735..07d2882894288c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/LogicalProperties.java @@ -43,17 +43,11 @@ public class LogicalProperties { protected final Supplier> outputMapSupplier; protected final Supplier> outputExprIdSetSupplier; protected final Supplier fdSupplier; - protected final Supplier> outputExpressionSupplier; private Integer hashCode = null; - public LogicalProperties(Supplier> outputSupplier, - Supplier fdSupplier, Supplier> outputExpressionSupplier) { - this(outputSupplier, fdSupplier, outputExpressionSupplier, ImmutableList::of); - } - public LogicalProperties(Supplier> outputSupplier, Supplier fdSupplier) { - this(outputSupplier, fdSupplier, null, ImmutableList::of); + this(outputSupplier, fdSupplier, ImmutableList::of); } /** @@ -64,7 +58,6 @@ public LogicalProperties(Supplier> outputSupplier, */ public LogicalProperties(Supplier> outputSupplier, Supplier fdSupplier, - Supplier> outputExpressionSupplier, Supplier> nonUserVisibleOutputSupplier) { this.outputSupplier = Suppliers.memoize( Objects.requireNonNull(outputSupplier, "outputSupplier can not be null") @@ -87,8 +80,6 @@ public LogicalProperties(Supplier> outputSupplier, this.fdSupplier = Suppliers.memoize( Objects.requireNonNull(fdSupplier, "FunctionalDependencies can not be null") ); - this.outputExpressionSupplier = Suppliers.memoize(Objects.requireNonNull(outputExpressionSupplier, - "outputExpressionSupplier can not be null")); } public List getOutput() { @@ -115,10 +106,6 @@ public List getOutputExprIds() { return outputExprIdsSupplier.get(); } - public List getOutputExpression() { - return outputExpressionSupplier.get(); - } - @Override public String toString() { return "LogicalProperties{" diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java index 780f536bdb69ce..4be6d35dc94692 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/AbstractPlan.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.properties.UnboundLogicalProperties; import org.apache.doris.nereids.trees.AbstractTreeNode; import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -156,11 +155,6 @@ public Set getOutputExprIdSet() { return getLogicalProperties().getOutputExprIdSet(); } - @Override - public List getOutputExpression() { - return getLogicalProperties().getOutputExpression(); - } - @Override public LogicalProperties getLogicalProperties() { // TODO: use bound()? @@ -187,8 +181,7 @@ public LogicalProperties computeLogicalProperties() { Supplier fdSupplier = () -> this instanceof LogicalPlan ? ((LogicalPlan) this).computeFuncDeps(outputSupplier) : FunctionalDependencies.EMPTY_FUNC_DEPS; - Supplier> outputExpressionSupplier = Suppliers.memoize(this::computeOutputExpression); - return new LogicalProperties(outputSupplier, fdSupplier, outputExpressionSupplier); + return new LogicalProperties(outputSupplier, fdSupplier); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java index 7bc494c2b7a693..eedfbbd8aa2ad2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.MutableState; @@ -92,11 +91,6 @@ public List getOutput() { return new ArrayList<>(); } - @Override - public List getOutputExpression() { - return new ArrayList<>(); - } - @Override public String treeString() { return "DUMMY"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index 0d3fa194e032f7..1a35fecaf29417 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -97,8 +97,6 @@ default boolean displayExtraPlanFirst() { */ List getOutput(); - List getOutputExpression(); - /** * Get output slot set of the plan. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java index 8a31c869348fda..d78308664b5e38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateViewCommand.java @@ -36,6 +36,7 @@ public CreateViewCommand(CreateViewInfo createViewInfo) { @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + createViewInfo.init(ctx); createViewInfo.validate(ctx); CreateViewStmt createViewStmt = createViewInfo.translateToLegacyStmt(ctx); Env.getCurrentEnv().createView(createViewStmt); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index efdb0c97a8fe08..09a9e1ba71483d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -48,9 +48,7 @@ import org.apache.doris.nereids.rules.analysis.BindRelationForCreateView; import org.apache.doris.nereids.rules.analysis.CheckPolicy; import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.BoundStar; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; @@ -60,7 +58,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; @@ -90,9 +87,8 @@ public class CreateViewInfo { private final List simpleColumnDefinitions; private final List finalCols = Lists.newArrayList(); private final List outputs = Lists.newArrayList(); - private final List projects = Lists.newArrayList(); private Plan analyzedPlan; - private Map, String> relationMap; + private Map, String> rewriteSqlMap; /** constructor*/ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String comment, LogicalPlan logicalQuery, @@ -108,6 +104,16 @@ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String commen this.simpleColumnDefinitions = simpleColumnDefinitions; } + /** init */ + public void init(ConnectContext ctx) throws UserException { + analyzeAndFillRewriteSqlMap(querySql, ctx); + OutermostPlanFinder outermostPlanFinder = new OutermostPlanFinder(); + AtomicReference outermostPlan = new AtomicReference<>(); + analyzedPlan.accept(outermostPlanFinder, outermostPlan); + outputs.addAll(outermostPlan.get().getOutput()); + createFinalCols(); + } + /**validate*/ public void validate(ConnectContext ctx) throws UserException { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); @@ -121,14 +127,6 @@ public void validate(ConnectContext ctx) throws UserException { viewName.getTbl(), PrivPredicate.CREATE)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE"); } - - analyzedPlan = parseAndAnalyzeView(querySql, ctx); - OutermostPlanFinder outermostPlanFinder = new OutermostPlanFinder(); - AtomicReference outermostPlan = new AtomicReference<>(); - analyzedPlan.accept(outermostPlanFinder, outermostPlan); - outputs.addAll(outermostPlan.get().getOutput()); - projects.addAll(outermostPlan.get().getOutputExpression()); - createFinalCols(); Set colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (Column col : finalCols) { if (!colSets.add(col.getName())) { @@ -145,23 +143,20 @@ public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { } CreateViewStmt createViewStmt = new CreateViewStmt(ifNotExists, viewName.transferToTableName(), cols, comment, null); - // expand star(*) in project list - Map, String> indexStringSqlMap = collectBoundStar(analyzedPlan); - indexStringSqlMap.putAll(relationMap); - String rewrittenSql = rewriteStarToColumn(indexStringSqlMap); + // expand star(*) in project list and replace table name with qualifier + String rewrittenSql = rewriteSql(rewriteSqlMap); // rewrite project alias - rewrittenSql = rewriteProjectsToUserDefineAlias2(rewrittenSql); + rewrittenSql = rewriteProjectsToUserDefineAlias(rewrittenSql); createViewStmt.setInlineViewDef(rewrittenSql); createViewStmt.setFinalColumns(finalCols); return createViewStmt; } - private Plan parseAndAnalyzeView(String ddlSql, ConnectContext ctx) { + private void analyzeAndFillRewriteSqlMap(String sql, ConnectContext ctx) { StatementContext stmtCtx = ctx.getStatementContext(); - LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(ddlSql); - // TODO: use a good to do this, such as eliminate UnboundResultSink + LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(sql); if (parsedViewPlan instanceof UnboundResultSink) { parsedViewPlan = (LogicalPlan) ((UnboundResultSink) parsedViewPlan).child(); } @@ -170,15 +165,16 @@ private Plan parseAndAnalyzeView(String ddlSql, ConnectContext ctx) { AnalyzerForCreateView analyzer = new AnalyzerForCreateView(viewContext, true); analyzer.analyze(); Plan planForRelation = viewContext.getRewritePlan(); - relationMap = collectUnBoundRelation(planForRelation); + rewriteSqlMap = collectUnBoundRelation(planForRelation); CascadesContext viewContextForStar = CascadesContext.initContext( stmtCtx, planForRelation, PhysicalProperties.ANY); AnalyzerForCreateView analyzerForStar = new AnalyzerForCreateView(viewContextForStar, false); analyzerForStar.analyze(); - return viewContextForStar.getRewritePlan(); + analyzedPlan = viewContextForStar.getRewritePlan(); + rewriteSqlMap.putAll(collectBoundStar(analyzedPlan)); } - private String rewriteStarToColumn(Map, String> indexStringSqlMap) { + private String rewriteSql(Map, String> indexStringSqlMap) { StringBuilder builder = new StringBuilder(); int beg = 0; for (Map.Entry, String> entry : indexStringSqlMap.entrySet()) { @@ -192,35 +188,6 @@ private String rewriteStarToColumn(Map, String> indexStri } private String rewriteProjectsToUserDefineAlias(String resSql) { - List projectExprs = Lists.newArrayList(); - for (int i = 0; i < projects.size(); i++) { - NamedExpression namedExpression = projects.get(i); - if (namedExpression instanceof Alias) { - projectExprs.add(namedExpression.child(0)); - } else { - projectExprs.add(namedExpression); - } - } - IndexFinder finder = new IndexFinder(); - ParserRuleContext tree = NereidsParser.toAst(resSql, DorisParser::singleStatement); - finder.visit(tree); - StringBuilder replaceWithColsBuilder = new StringBuilder(); - for (int i = 0; i < projectExprs.size(); ++i) { - replaceWithColsBuilder.append(projectExprs.get(i).toSql()); - replaceWithColsBuilder.append(" AS `"); - String escapeBacktick = finalCols.get(i).getName().replace("`", "``"); - replaceWithColsBuilder.append(escapeBacktick); - replaceWithColsBuilder.append('`'); - if (i != projectExprs.size() - 1) { - replaceWithColsBuilder.append(", "); - } - } - String replaceWithCols = replaceWithColsBuilder.toString(); - return StringUtils.overlay(resSql, replaceWithCols, finder.getIndex().first, - finder.getIndex().second + 1); - } - - private String rewriteProjectsToUserDefineAlias2(String resSql) { IndexFinder finder = new IndexFinder(); ParserRuleContext tree = NereidsParser.toAst(resSql, DorisParser::singleStatement); finder.visit(tree); @@ -324,14 +291,6 @@ public Void visitLogicalCTEProducer(LogicalCTEProducer cteProduc AtomicReference target) { return null; } - - @Override - public Void visitLogicalSetOperation(LogicalSetOperation setOperation, AtomicReference target) { - if (found) { - return null; - } - return super.visit(setOperation, target); - } } /** traverse ast to find the outermost project list location information in sql*/ @@ -368,7 +327,6 @@ public Void visitSelectColumnClause(DorisParser.SelectColumnClauseContext ctx) { stopIndex = ctx.getStop().getStopIndex(); found = true; - // 这里默认*已经被替换了，不存在*或者* except NamedExpressionSeqContext namedExpressionSeqContext = ctx.namedExpressionSeq(); namedExpressionContexts = namedExpressionSeqContext.namedExpression(); return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java index 900add75f89e1f..20647a3808ebe7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAggregate.java @@ -390,9 +390,4 @@ public ImmutableSet computeFdItems(Supplier> outputSupplier) return builder.build(); } - - @Override - public List computeOutputExpression() { - return outputExpressions; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java index 2472247f1d1c64..517048c209aba4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InSubquery; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; @@ -162,17 +161,6 @@ public List computeOutput() { .build(); } - @Override - public List computeOutputExpression() { - return ImmutableList.builder() - .addAll(left().getOutputExpression()) - .addAll(markJoinSlotReference.isPresent() - ? ImmutableList.of(markJoinSlotReference.get()) : ImmutableList.of()) - .addAll(needAddSubOutputToProjects - ? ImmutableList.of(right().getOutputExpression().get(0)) : ImmutableList.of()) - .build(); - } - @Override public String toString() { return Utils.toSqlString("LogicalApply", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java index 6abcba3ebb26d9..0279c9701caaff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalAssertNumRows.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.AssertNumRowsElement; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -119,10 +118,4 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr public List computeOutput() { return child().getOutput().stream().map(o -> o.withNullable(true)).collect(Collectors.toList()); } - - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java index 8b6904ca8d69e6..7a75dd5c1a4abf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTE.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -71,11 +70,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - @Override public String toString() { return Utils.toSqlString("LogicalCTE", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java index 321dadaa174037..67aef52f979ead 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEAnchor.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -84,11 +83,6 @@ public List computeOutput() { return right().getOutput(); } - @Override - public List computeOutputExpression() { - return right().getOutputExpression(); - } - public CTEId getCteId() { return cteId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java index 2aa6bf4ee8555e..71b1c43f791191 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; @@ -146,11 +145,6 @@ public List computeOutput() { return ImmutableList.copyOf(producerToConsumerOutputMap.values()); } - @Override - public List computeOutputExpression() { - return ImmutableList.copyOf(producerToConsumerOutputMap.values()); - } - public CTEId getCteId() { return cteId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java index 8fab3a1d427ae8..aeae13302d9245 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEProducer.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -91,11 +90,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - @Override public String toString() { return Utils.toSqlString("LogicalCteProducer[" + id.asInt() + "]", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 380d9a30c4d711..4076e8348e2208 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -101,14 +101,6 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } - @Override - public List computeOutputExpression() { - return table.getBaseSchema() - .stream() - .map(col -> SlotReference.fromColumn(table, col, qualified(), this)) - .collect(ImmutableList.toImmutableList()); - } - public List getQualifier() { return qualifier; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java index 800a2a3ccd3a49..bda3b1f49d570d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java @@ -80,11 +80,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - @Override public String toString() { return Utils.toSqlString("LogicalCheckPolicy"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java index 23b98e76d7aad6..a49e37d8df40a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; @@ -111,14 +110,6 @@ public List computeOutput() { .build(); } - @Override - public List computeOutputExpression() { - return ImmutableList.builder() - .addAll(logicalOlapScan.getOutput()) - .add(columnIdSlot) - .build(); - } - @Override public Plan withGroupExpression(Optional groupExpression) { return new LogicalDeferMaterializeOlapScan(logicalOlapScan, deferMaterializeSlotIds, columnIdSlot, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java index 57b284aa1678ff..5bf0afe1faef59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeTopN.java @@ -27,7 +27,6 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; @@ -120,13 +119,6 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } - @Override - public List computeOutputExpression() { - return logicalTopN.getOutputExpression().stream() - .filter(s -> !(s.getExprId().equals(columnIdSlot.getExprId()))) - .collect(ImmutableList.toImmutableList()); - } - @Override public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { FunctionalDependencies fd = child(0).getLogicalProperties().getFunctionalDependencies(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java index 9555ccc2fab9dc..ab816f7a40859d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEmptyRelation.java @@ -100,11 +100,6 @@ public List getOutputs() { return projects; } - @Override - public List getOutputExpression() { - return projects; - } - @Override public Plan pruneOutputs(List prunedOutputs) { return withProjects(prunedOutputs); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java index 907425d8e48c7d..4375124a22455d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFilter.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; @@ -88,11 +87,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - @Override public String toString() { return Utils.toSqlString("LogicalFilter[" + id.asInt() + "]", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java index 2caa9e87d719f5..0fb3964e511f0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalGenerate.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.plans.Plan; @@ -131,14 +130,6 @@ public List computeOutput() { .build(); } - @Override - public List computeOutputExpression() { - return ImmutableList.builder() - .addAll(child().getOutput()) - .addAll(generatorOutput) - .build(); - } - @Override public String toString() { return Utils.toSqlString("LogicalGenerate", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java index dafa23406f2922..e4d01f127482b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHaving.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -103,11 +102,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - @Override public int hashCode() { return Objects.hashCode(conjuncts); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java index 009bbe89006871..b2a2a1d83ca3e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java @@ -82,11 +82,6 @@ public List computeOutput() { return ImmutableList.of(); } - @Override - public List computeOutputExpression() { - return ImmutableList.of(); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index 07282352cf8b47..d18a454fe73296 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -34,7 +34,6 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.DistributeType; import org.apache.doris.nereids.trees.plans.JoinType; @@ -265,15 +264,6 @@ public List computeOutput() { .build(); } - @Override - public List computeOutputExpression() { - return ImmutableList.builder() - .addAll(JoinUtils.getJoinOutput(joinType, left(), right())) - .addAll(isMarkJoin() - ? ImmutableList.of(markJoinSlotReference.get()) : ImmutableList.of()) - .build(); - } - @Override public String toString() { List args = Lists.newArrayList( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java index c56f7385ddf154..482c7e26058f25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalLimit.java @@ -25,7 +25,6 @@ import org.apache.doris.nereids.properties.FunctionalDependencies.Builder; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.LimitPhase; @@ -91,11 +90,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - @Override public String toString() { return Utils.toSqlString("LogicalLimit", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index f5d4e6e4a7e2c6..cb5d1847fef2a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule; import org.apache.doris.nereids.trees.TableSample; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; @@ -349,11 +348,6 @@ public List computeOutput() { return slots.build(); } - @Override - public List computeOutputExpression() { - return computeOutput().stream().map(slot -> (NamedExpression) slot).collect(Collectors.toList()); - } - @Override public Set getInputRelations() { Set relationIdSet = Sets.newHashSet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index 0a77933f22ab74..d338be5cc88185 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -94,11 +94,6 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } - @Override - public List computeOutputExpression() { - return projects; - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java index 9d246860b5d270..69da96e8f7ebcb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalPartitionTopN.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.WindowExpression; @@ -112,11 +111,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - public WindowFuncType getFunction() { return function; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 003121e5ed9645..4e19c5ca623efd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -275,9 +275,4 @@ public List getBoundStars() { public void setBoundStars(List boundStars) { this.boundStars = boundStars; } - - @Override - public List computeOutputExpression() { - return projects; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index ff738311182993..5e0311afe1b518 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -108,11 +108,6 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } - @Override - public List computeOutputExpression() { - return outputExpressions; - } - @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalRepeat(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java index e974a75b00ce0f..127889ea7ed471 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSelectHint.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.SelectHint; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; @@ -106,11 +105,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - @Override public String toString() { String hintStr = this.hints.entrySet() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java index f8c3fa53305772..2e4ddb55ff2f02 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -105,11 +105,6 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } - @Override - public List computeOutputExpression() { - return outputs; - } - public List> collectChildrenProjections() { return castCommonDataTypeOutputs(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java index 3da49ac7f99c2d..ea2fa49e4664bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSink.java @@ -66,11 +66,6 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } - @Override - public List computeOutputExpression() { - return outputExprs; - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java index 411372c20aac7c..9d9d321e659636 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -67,11 +66,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - public List getOrderKeys() { return orderKeys; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index 16d41e30c09cdb..68a451f1df1b19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -43,7 +42,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import java.util.stream.Collectors; /** * The node of logical plan for sub query and alias @@ -101,11 +99,6 @@ public List computeOutput() { return currentOutput.build(); } - @Override - public List computeOutputExpression() { - return computeOutput().stream().map(slot -> (NamedExpression) slot).collect(Collectors.toList()); - } - public String getAlias() { return qualifier.get(qualifier.size() - 1); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java index 3a728e69534aa8..8302c9345dcc17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java @@ -19,7 +19,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; @@ -37,7 +36,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; /** LogicalTableValuedFunctionRelation */ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation, BlockFuncDepsPropagation { @@ -105,11 +103,6 @@ public List computeOutput() { .collect(ImmutableList.toImmutableList()); } - @Override - public List computeOutputExpression() { - return computeOutput().stream().map(slot -> (NamedExpression) slot).collect(Collectors.toList()); - } - @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalTVFRelation(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java index f57f7e077970d8..63e3dc9c0b9717 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTopN.java @@ -26,7 +26,6 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; @@ -73,11 +72,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - @Override public List getOrderKeys() { return orderKeys; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java index 43485788b03bfe..e8621b1754c757 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.properties.FunctionalDependencies; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -129,11 +128,6 @@ public List computeOutput() { return child().getOutput(); } - @Override - public List computeOutputExpression() { - return child().getOutputExpression(); - } - @Override public FunctionalDependencies computeFuncDeps(Supplier> outputSupplier) { return ((LogicalPlan) child()).computeFuncDeps(outputSupplier); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java index ac9d2294d3308a..457678787bd738 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalWindow.java @@ -144,15 +144,6 @@ public List computeOutput() { .build(); } - @Override - public List computeOutputExpression() { - return new ImmutableList.Builder() - .addAll(child().getOutputExpression()) - .addAll(windowExpressions.stream() - .collect(ImmutableList.toImmutableList())) - .build(); - } - @Override public String toString() { return Utils.toSqlString("LogicalWindow", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index 790855cb26efdd..a73c183e4e75c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -30,7 +30,6 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapContains; @@ -389,24 +388,6 @@ public static List getJoinOutput(JoinType joinType, Plan left, Plan right) } } - /** getJoinOutputExpression */ - public static List getJoinOutputExpression(JoinType joinType, Plan left, Plan right) { - switch (joinType) { - case LEFT_SEMI_JOIN: - case LEFT_ANTI_JOIN: - case NULL_AWARE_LEFT_ANTI_JOIN: - return ImmutableList.copyOf(left.getOutputExpression()); - case RIGHT_SEMI_JOIN: - case RIGHT_ANTI_JOIN: - return ImmutableList.copyOf(right.getOutputExpression()); - default: - return ImmutableList.builder() - .addAll(left.getOutputExpression()) - .addAll(right.getOutputExpression()) - .build(); - } - } - public static boolean hasMarkConjuncts(Join join) { return !join.getMarkJoinConjuncts().isEmpty(); } diff --git a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out deleted file mode 100644 index d88325d3711d83..00000000000000 --- a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out +++ /dev/null @@ -1,46 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !show -- -show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); - --- !select -- -1 1 30 -1 2 5 -1 3 10 -1 300 2 -2 1 100 -2 2 10 -2 3 44 -2 200 1111 -3 1 10 -3 2 1 -23 900 1 -100 100 1 -200 20 1 -300 20 1 - --- !select -- -1 5 -2 10 -3 1 - --- !show -- -show_create_table_and_views_view CREATE VIEW `show_create_table_and_views_view` COMMENT 'VIEW' AS SELECT user_id AS `user_id`, cost AS `cost` FROM `internal`.`show_create_table_and_views_db`.`show_create_table_and_views_table`\n WHERE good_id = 2; utf8mb4 utf8mb4_0900_bin - --- !select -- -1 47 -2 1265 -3 11 -23 1 -100 1 -200 1 -300 1 - --- !show -- -show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); - --- !show -- -show_create_table_and_views_like CREATE TABLE `show_create_table_and_views_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); - --- !show -- -show_create_table_and_views_like_with_rollup CREATE TABLE `show_create_table_and_views_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); - diff --git a/regression-test/suites/show_p0/test_show_create_table_and_views.groovy b/regression-test/suites/show_p0/test_show_create_table_and_views.groovy index 2814ba8cd8ac29..6c6322e5c4aa03 100644 --- a/regression-test/suites/show_p0/test_show_create_table_and_views.groovy +++ b/regression-test/suites/show_p0/test_show_create_table_and_views.groovy @@ -16,7 +16,6 @@ // under the License. suite("test_show_create_table_and_views", "show") { - sql "SET enable_nereids_planner=false;" def ret = sql "SHOW FRONTEND CONFIG like '%enable_feature_binlog%';" logger.info("${ret}") if (ret.size() != 0 && ret[0].size() > 1 && ret[0][1] == 'false') { diff --git a/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy deleted file mode 100644 index aa235b2b188afd..00000000000000 --- a/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy +++ /dev/null @@ -1,126 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_show_create_table_and_views_nereids", "show") { - sql "SET enable_nereids_planner=true;" - sql "SET enable_fallback_to_original_planner=false;" - def ret = sql "SHOW FRONTEND CONFIG like '%enable_feature_binlog%';" -// logger.info("${ret}") -// if (ret.size() != 0 && ret[0].size() > 1 && ret[0][1] == 'false') { -// logger.info("enable_feature_binlog=false in frontend config, no need to run this case.") -// return -// } - - String suiteName = "show_create_table_and_views" - String dbName = "${suiteName}_db" - String tableName = "${suiteName}_table" - String viewName = "${suiteName}_view" - String rollupName = "${suiteName}_rollup" - String likeName = "${suiteName}_like" - - sql "CREATE DATABASE IF NOT EXISTS ${dbName}" - sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" - sql """ - CREATE TABLE ${dbName}.${tableName} ( - `user_id` LARGEINT NOT NULL, - `good_id` LARGEINT NOT NULL, - `cost` BIGINT SUM DEFAULT "0" - ) - AGGREGATE KEY(`user_id`, `good_id`) - PARTITION BY RANGE(`good_id`) - ( - PARTITION p1 VALUES LESS THAN ("100"), - PARTITION p2 VALUES LESS THAN ("200"), - PARTITION p3 VALUES LESS THAN ("300"), - PARTITION p4 VALUES LESS THAN ("400"), - PARTITION p5 VALUES LESS THAN ("500"), - PARTITION p6 VALUES LESS THAN ("600"), - PARTITION p7 VALUES LESS THAN MAXVALUE - ) - DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 - PROPERTIES ( - "replication_num" = "1" - ) - """ - - sql """INSERT INTO ${dbName}.${tableName} VALUES - (1, 1, 10), - (1, 1, 20), - (1, 2, 5), - (1, 3, 10), - (2, 1, 0), - (2, 1, 100), - (3, 1, 10), - (2, 2, 10), - (2, 3, 44), - (3, 2, 1), - (100, 100, 1), - (200, 20, 1), - (300, 20, 1), - (1, 300, 2), - (2, 200, 1111), - (23, 900, 1)""" - - qt_show "SHOW CREATE TABLE ${dbName}.${tableName}" - qt_select "SELECT * FROM ${dbName}.${tableName} ORDER BY user_id, good_id" - - // create view and show - sql """ - CREATE VIEW IF NOT EXISTS ${dbName}.${viewName} (user_id, cost) - AS - SELECT user_id, cost FROM ${dbName}.${tableName} - WHERE good_id = 2 - """ - qt_select "SELECT * FROM ${dbName}.${viewName} ORDER BY user_id" - qt_show "SHOW CREATE VIEW ${dbName}.${viewName}" - - // create rollup - sql """ALTER TABLE ${dbName}.${tableName} - ADD ROLLUP ${rollupName} (user_id, cost) - """ - - def isAlterTableFinish = { -> - def records = sql """SHOW ALTER TABLE ROLLUP FROM ${dbName}""" - for (def row in records) { - if (row[5] == "${rollupName}" && row[8] == "FINISHED") { - return true - } - } - false - } - while (!isAlterTableFinish()) { - Thread.sleep(100) - } - - qt_select "SELECT user_id, SUM(cost) FROM ${dbName}.${tableName} GROUP BY user_id ORDER BY user_id" - qt_show "SHOW CREATE TABLE ${dbName}.${tableName}" - - // create like - sql "CREATE TABLE ${dbName}.${likeName} LIKE ${dbName}.${tableName}" - qt_show "SHOW CREATE TABLE ${dbName}.${likeName}" - - // create like with rollup - sql "CREATE TABLE ${dbName}.${likeName}_with_rollup LIKE ${dbName}.${tableName} WITH ROLLUP" - qt_show "SHOW CREATE TABLE ${dbName}.${likeName}_with_rollup" - - sql "DROP TABLE IF EXISTS ${dbName}.${likeName}_with_rollup FORCE" - sql "DROP TABLE ${dbName}.${likeName} FORCE" - sql "DROP VIEW ${dbName}.${viewName}" - sql "DROP TABLE ${dbName}.${tableName} FORCE" - sql "DROP DATABASE ${dbName} FORCE" -} - From 32ba0a7b1fc62548e2da1bc36e317f6af18b52ee Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 1 Apr 2024 14:39:12 +0800 Subject: [PATCH 15/23] [feat](nereids) support create view in nereids --- .../trees/expressions/SlotReference.java | 1 - .../data/ddl_p0/test_create_view_nereids.out | 22 +++++++++++++++++++ .../ddl_p0/test_create_view_nereids.groovy | 8 +++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index 2c7df32575791f..c63855157d28d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -188,7 +188,6 @@ public Optional getTable() { @Override public String toSql() { - // return getQualifiedNameWithBacktick(); return name.get(); } diff --git a/regression-test/data/ddl_p0/test_create_view_nereids.out b/regression-test/data/ddl_p0/test_create_view_nereids.out index 09a573593d6d37..a7795e15e1150d 100644 --- a/regression-test/data/ddl_p0/test_create_view_nereids.out +++ b/regression-test/data/ddl_p0/test_create_view_nereids.out @@ -138,3 +138,25 @@ test_view_expression CREATE VIEW `test_view_expression` COMMENT 'VIEW' AS (selec -- !test_alias_sql -- test_view_alias CREATE VIEW `test_view_alias` COMMENT 'VIEW' AS (\n select c8 AS `c1`, c2 AS `c2`, c1 AS `c3` from (select a+1 c8,abs(a)+2+1 as c2, cast(b as varchar(10)) as c1 from `internal`.`regression_test_ddl_p0`.`mal_test_view`) t); utf8mb4 utf8mb4_0900_bin +-- !test_star_except -- +6 \N 6 +1 1 2 +2 1 3 +1 1 4 +2 1 4 +2 1 7 +3 2 8 +2 3 5 +1 3 6 +2 3 9 +2 4 2 +3 5 \N +3 5 6 +3 5 6 +4 5 6 +3 5 8 +6 7 1 + +-- !test_star_except_sql -- +test_view_star_except CREATE VIEW `test_view_star_except` COMMENT 'VIEW' AS select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from `internal`.`regression_test_ddl_p0`.`mal_test_view`; utf8mb4 utf8mb4_0900_bin + diff --git a/regression-test/suites/ddl_p0/test_create_view_nereids.groovy b/regression-test/suites/ddl_p0/test_create_view_nereids.groovy index e5a9c56ee4e2a8..33aa19239dc2f1 100644 --- a/regression-test/suites/ddl_p0/test_create_view_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_view_nereids.groovy @@ -252,4 +252,12 @@ suite("test_create_view_nereids") { qt_test_alias "select * from test_view_alias order by c1,c2,c3;" qt_test_alias_sql "show create view test_view_alias;" + // test * except + sql "drop view if exists test_view_star_except;" + sql """ + create view test_view_star_except as select * except(pk) from mal_test_view; + """ + qt_test_star_except "select * from test_view_star_except order by a, b;" + qt_test_star_except_sql "show create view test_view_star_except;" + } From c8ba8260609d8dd37d80ef51bda259335a0fc565 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 1 Apr 2024 16:25:16 +0800 Subject: [PATCH 16/23] [feat](nereids) support create view in nereids --- .../org/apache/doris/nereids/rules/analysis/BindRelation.java | 3 --- .../main/java/org/apache/doris/nereids/trees/plans/Plan.java | 4 ---- 2 files changed, 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 9fafe361a0877f..22eb74fd709de7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -42,10 +42,8 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.plans.Plan; @@ -59,7 +57,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java index 226a95cac88888..d73b7390ce8d59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java @@ -158,10 +158,6 @@ default List computeOutput() { throw new IllegalStateException("Not support compute output for " + getClass().getName()); } - default List computeOutputExpression() { - throw new IllegalStateException("Not support compute output expression for " + getClass().getName()); - } - /** * Get the input relation ids set of the plan. * @return The result is collected from all inputs relations From c9ab703f3abe033c920438766274a8e786165d3f Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 1 Apr 2024 22:15:08 +0800 Subject: [PATCH 17/23] [feat](nereids) support create view in nereids --- .../nereids/parser/LogicalPlanBuilder.java | 36 +++++++--- .../doris/nereids/parser/NereidsParser.java | 12 ++++ .../nereids/rules/analysis/BindRelation.java | 57 +++++++++++---- .../analysis/BindRelationForCreateView.java | 63 +---------------- .../nereids/trees/expressions/BoundStar.java | 9 ++- .../plans/commands/info/CreateViewInfo.java | 70 ++++++++----------- .../plans/logical/LogicalCatalogRelation.java | 10 +++ .../plans/logical/LogicalSubQueryAlias.java | 11 +++ .../data/ddl_p0/test_create_view_nereids.out | 40 ++++++++--- .../ddl_p0/test_create_view_nereids.groovy | 6 ++ 10 files changed, 175 insertions(+), 139 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 7d3a791bebe3db..81e364ba0c98b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -475,6 +475,15 @@ */ @SuppressWarnings({"OptionalUsedAsFieldOrParameterType", "OptionalGetWithoutIsPresent"}) public class LogicalPlanBuilder extends DorisParserBaseVisitor { + private final boolean forCreateView; + + public LogicalPlanBuilder() { + forCreateView = false; + } + + public LogicalPlanBuilder(boolean forCreateView) { + this.forCreateView = forCreateView; + } @SuppressWarnings("unchecked") protected T typedVisit(ParseTree ctx) { @@ -1334,11 +1343,14 @@ public LogicalPlan visitTableName(TableNameContext ctx) { MultipartIdentifierContext identifier = ctx.multipartIdentifier(); TableSample tableSample = ctx.sample() == null ? null : (TableSample) visit(ctx.sample()); - LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy( + UnboundRelation relation = forCreateView ? new UnboundRelation(StatementScopeIdGenerator.newRelationId(), + tableId, partitionNames, isTempPart, tabletIdLists, relationHints, + Optional.ofNullable(tableSample), indexName, scanParams, + Pair.of(identifier.start.getStartIndex(), identifier.stop.getStopIndex())) : new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, - Optional.ofNullable(tableSample), indexName, scanParams, - Pair.of(identifier.start.getStartIndex(), identifier.stop.getStopIndex()))); + Optional.ofNullable(tableSample), indexName, scanParams); + LogicalPlan checkedRelation = LogicalPlanBuilderAssistant.withCheckPolicy(relation); LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias()); for (LateralViewContext lateralViewContext : ctx.lateralView()) { plan = withGenerate(plan, lateralViewContext); @@ -1387,7 +1399,9 @@ public Expression visitStar(StarContext ctx) { } else { target = ImmutableList.of(); } - return new UnboundStar(target, Pair.of(ctx.start.getStartIndex(), ctx.stop.getStopIndex())); + return forCreateView + ? new UnboundStar(target, Pair.of(ctx.start.getStartIndex(), ctx.stop.getStopIndex())) : + new UnboundStar(target); }); } @@ -2876,9 +2890,10 @@ private LogicalPlan withSelectQuerySpecification( if (!expressions.stream().allMatch(UnboundSlot.class::isInstance)) { throw new ParseException("only column name is supported in except clause", selectColumnCtx); } - project = new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of(), - Pair.of(selectColumnCtx.start.getStartIndex(), selectColumnCtx.stop.getStopIndex()))), - expressions, isDistinct, aggregate); + UnboundStar star = forCreateView ? new UnboundStar(ImmutableList.of(), + Pair.of(selectColumnCtx.start.getStartIndex(), selectColumnCtx.stop.getStopIndex())) : + new UnboundStar(ImmutableList.of()); + project = new LogicalProject<>(ImmutableList.of(star), expressions, isDistinct, aggregate); } else { List projects = getNamedExpressions(selectColumnCtx.namedExpressionSeq()); project = new LogicalProject<>(projects, ImmutableList.of(), isDistinct, aggregate); @@ -3057,9 +3072,10 @@ private LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContext if (!expressions.stream().allMatch(UnboundSlot.class::isInstance)) { throw new ParseException("only column name is supported in except clause", selectCtx); } - return new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of(), - Pair.of(selectCtx.start.getStartIndex(), selectCtx.stop.getStopIndex()))), - expressions, isDistinct, input); + UnboundStar star = forCreateView ? new UnboundStar(ImmutableList.of(), + Pair.of(selectCtx.start.getStartIndex(), selectCtx.stop.getStopIndex())) : + new UnboundStar(ImmutableList.of()); + return new LogicalProject<>(ImmutableList.of(star), expressions, isDistinct, input); } else { List projects = getNamedExpressions(selectCtx.namedExpressionSeq()); return new LogicalProject<>(projects, Collections.emptyList(), isDistinct, input); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index d40b45d41a4042..da6881a4905468 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -164,6 +164,18 @@ private T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder, return (T) realLogicalPlanBuilder.visit(tree); } + public LogicalPlan parseForCreateView(String sql) { + return parseForCreateViewInternal(sql, null, DorisParser::singleStatement); + } + + private T parseForCreateViewInternal(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder, + Function parseFunction) { + ParserRuleContext tree = toAst(sql, parseFunction); + LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null + ? new LogicalPlanBuilder(true) : logicalPlanBuilder; + return (T) realLogicalPlanBuilder.visit(tree); + } + /** toAst */ public static ParserRuleContext toAst(String sql, Function parseFunction) { DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 22eb74fd709de7..7bc09d902cc43a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -215,6 +215,7 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, unboundRelation.getTableSample()); } } + scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign() && !ConnectContext.get().getSessionVariable().skipDeleteSign() && !scan.isDirectMvScan()) { @@ -246,39 +247,65 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio case OLAP: case MATERIALIZED_VIEW: return makeOlapScan(table, unboundRelation, qualifierWithoutTableName); - case VIEW: + case VIEW: { View view = (View) table; String inlineViewDef = view.getInlineViewDef(); Plan viewBody = parseAndAnalyzeView(inlineViewDef, cascadesContext); LogicalView logicalView = new LogicalView<>(view, viewBody); - return new LogicalSubQueryAlias<>(tableQualifier, logicalView); - case HMS_EXTERNAL_TABLE: + LogicalSubQueryAlias alias = new LogicalSubQueryAlias<>(tableQualifier, logicalView); + alias.setIndexInSqlString(unboundRelation.getIndexInSqlString()); + return alias; + } + case HMS_EXTERNAL_TABLE: { HMSExternalTable hmsTable = (HMSExternalTable) table; if (Config.enable_query_hive_views && hmsTable.isView()) { String hiveCatalog = hmsTable.getCatalog().getName(); String ddlSql = hmsTable.getViewText(); Plan hiveViewPlan = parseAndAnalyzeHiveView(hiveCatalog, ddlSql, cascadesContext); - return new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan); + LogicalSubQueryAlias alias = new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan); + alias.setIndexInSqlString(unboundRelation.getIndexInSqlString()); + return alias; } hmsTable.setScanParams(unboundRelation.getScanParams()); - return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, + LogicalFileScan scan = new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, qualifierWithoutTableName, unboundRelation.getTableSample()); + scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); + return scan; + } case ICEBERG_EXTERNAL_TABLE: case PAIMON_EXTERNAL_TABLE: case MAX_COMPUTE_EXTERNAL_TABLE: - case TRINO_CONNECTOR_EXTERNAL_TABLE: - return new LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table, + case TRINO_CONNECTOR_EXTERNAL_TABLE: { + LogicalFileScan scan = new LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table, qualifierWithoutTableName, unboundRelation.getTableSample()); - case SCHEMA: - return new LogicalSchemaScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); + scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); + return scan; + } + case SCHEMA: { + LogicalSchemaScan scan = new LogicalSchemaScan(unboundRelation.getRelationId(), table, + qualifierWithoutTableName); + scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); + return scan; + } case JDBC_EXTERNAL_TABLE: - case JDBC: - return new LogicalJdbcScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); - case ODBC: - return new LogicalOdbcScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); - case ES_EXTERNAL_TABLE: - return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, + case JDBC: { + LogicalJdbcScan scan = new LogicalJdbcScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); + scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); + return scan; + } + case ODBC: { + LogicalOdbcScan scan = new LogicalOdbcScan(unboundRelation.getRelationId(), table, + qualifierWithoutTableName); + scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); + return scan; + } + case ES_EXTERNAL_TABLE: { + LogicalEsScan scan = new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, + qualifierWithoutTableName); + scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); + return scan; + } case TEST_EXTERNAL_TABLE: return new LogicalTestScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); default: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java index 3cb5fdc29f09cc..a29d8754dfd0b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java @@ -17,85 +17,26 @@ package org.apache.doris.nereids.rules.analysis; -import org.apache.doris.common.Pair; -import org.apache.doris.nereids.CTEContext; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.analyzer.Unbound; import org.apache.doris.nereids.analyzer.UnboundRelation; -import org.apache.doris.nereids.hint.LeadingHint; import org.apache.doris.nereids.pattern.MatchingContext; -import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.algebra.Relation; -import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.RelationUtil; import java.util.List; -import java.util.Optional; /** BindRelationForCreateView */ public class BindRelationForCreateView extends OneAnalysisRuleFactory { - - // TODO: cte will be copied to a sub-query with different names but the id of the unbound relation in them - // are the same, so we use new relation id when binding relation, and will fix this bug later. @Override public Rule build() { - return unboundRelation().thenApply(ctx -> { - Plan plan = doBindRelation(ctx); - if (!(plan instanceof Unbound) && plan instanceof Relation) { - // init output and allocate slot id immediately, so that the slot id increase - // in the order in which the table appears. - LogicalProperties logicalProperties = plan.getLogicalProperties(); - logicalProperties.getOutput(); - } - return plan; - }).toRule(RuleType.BINDING_RELATION_FOR_CREATE_VIEW); + return unboundRelation().thenApply(this::doBindRelation).toRule(RuleType.BINDING_RELATION_FOR_CREATE_VIEW); } private Plan doBindRelation(MatchingContext ctx) { - List nameParts = ctx.root.getNameParts(); - switch (nameParts.size()) { - case 1: { // table - // Use current database name from catalog. - return bindWithCurrentDb(ctx.cascadesContext, ctx.root); - } - case 2: - // db.table - // Use database name from table name parts. - case 3: { - // catalog.db.table - // Use catalog and database name from name parts. - return bind(ctx.cascadesContext, ctx.root); - } - default: - throw new IllegalStateException("Table name [" + ctx.root.getTableName() + "] is invalid."); - } - } - - private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRelation unboundRelation) { - String tableName = unboundRelation.getNameParts().get(0); - // check if it is a CTE's name - CTEContext cteContext = cascadesContext.getCteContext().findCTEContext(tableName).orElse(null); - if (cteContext != null) { - Optional analyzedCte = cteContext.getAnalyzedCTEPlan(tableName); - if (analyzedCte.isPresent()) { - LogicalCTEConsumer consumer = new LogicalCTEConsumer(unboundRelation.getRelationId(), - cteContext.getCteId(), tableName, analyzedCte.get()); - if (cascadesContext.isLeadingJoin()) { - LeadingHint leading = (LeadingHint) cascadesContext.getHintMap().get("Leading"); - leading.putRelationIdAndTableName(Pair.of(consumer.getRelationId(), tableName)); - leading.getRelationIdToScanMap().put(consumer.getRelationId(), consumer); - } - return consumer; - } - } - List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), - unboundRelation.getNameParts()); - unboundRelation.setTableQualifier(tableQualifier); - return unboundRelation; + return bind(ctx.cascadesContext, ctx.root); } private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboundRelation) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java index 5182894348826f..781c53ad65f0da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java @@ -61,10 +61,13 @@ public String toSqlWithBacktick() { /** toSqlWithBacktick */ public String toSqlWithBacktick(List excepts) { List outputs = Lists.newArrayList(); - Set exceptSet = Sets.newHashSet(); - exceptSet.addAll(excepts); + Set exceptSet = Sets.newHashSet(); + for (NamedExpression except : excepts) { + exceptSet.add(except.getName()); + } for (Expression child : children) { - if (!exceptSet.contains(child)) { + SlotReference slot = (SlotReference) child; + if (!exceptSet.contains(slot.getName())) { outputs.add(child); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index 09a9e1ba71483d..f431ce6b8fe353 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -35,7 +35,6 @@ import org.apache.doris.nereids.DorisParserBaseVisitor; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.jobs.executor.AbstractBatchJobExecutor; @@ -45,7 +44,6 @@ import org.apache.doris.nereids.rules.analysis.AnalyzeCTE; import org.apache.doris.nereids.rules.analysis.BindExpression; import org.apache.doris.nereids.rules.analysis.BindRelation; -import org.apache.doris.nereids.rules.analysis.BindRelationForCreateView; import org.apache.doris.nereids.rules.analysis.CheckPolicy; import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint; import org.apache.doris.nereids.trees.expressions.BoundStar; @@ -55,9 +53,11 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; @@ -156,22 +156,16 @@ public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { private void analyzeAndFillRewriteSqlMap(String sql, ConnectContext ctx) { StatementContext stmtCtx = ctx.getStatementContext(); - LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(sql); + LogicalPlan parsedViewPlan = new NereidsParser().parseForCreateView(sql); if (parsedViewPlan instanceof UnboundResultSink) { parsedViewPlan = (LogicalPlan) ((UnboundResultSink) parsedViewPlan).child(); } - CascadesContext viewContext = CascadesContext.initContext( - stmtCtx, parsedViewPlan, PhysicalProperties.ANY); - AnalyzerForCreateView analyzer = new AnalyzerForCreateView(viewContext, true); - analyzer.analyze(); - Plan planForRelation = viewContext.getRewritePlan(); - rewriteSqlMap = collectUnBoundRelation(planForRelation); CascadesContext viewContextForStar = CascadesContext.initContext( - stmtCtx, planForRelation, PhysicalProperties.ANY); - AnalyzerForCreateView analyzerForStar = new AnalyzerForCreateView(viewContextForStar, false); + stmtCtx, parsedViewPlan, PhysicalProperties.ANY); + AnalyzerForCreateView analyzerForStar = new AnalyzerForCreateView(viewContextForStar); analyzerForStar.analyze(); analyzedPlan = viewContextForStar.getRewritePlan(); - rewriteSqlMap.putAll(collectBoundStar(analyzedPlan)); + rewriteSqlMap = collectStarAndRelationInfo(analyzedPlan); } private String rewriteSql(Map, String> indexStringSqlMap) { @@ -233,32 +227,38 @@ private void createFinalCols() throws org.apache.doris.common.AnalysisException } } - private Map, String> collectUnBoundRelation(Plan plan) { - TreeMap, String> result = new TreeMap<>(new Pair.PairComparator<>()); - plan.foreach(node -> { - if (node instanceof UnboundRelation) { - UnboundRelation unboundRelation = (UnboundRelation) node; - List tableQualifier = unboundRelation.getTableQualifier(); - result.put(unboundRelation.getIndexInSqlString(), Utils.qualifiedNameWithBacktick(tableQualifier)); - } - }); - return result; - } - - private Map, String> collectBoundStar(Plan plan) { + private Map, String> collectStarAndRelationInfo(Plan plan) { TreeMap, String> result = new TreeMap<>(new Pair.PairComparator<>()); plan.foreach(node -> { if (node instanceof LogicalProject) { LogicalProject project = (LogicalProject) node; if (project.getExcepts().isEmpty()) { for (BoundStar star : project.getBoundStars()) { + if (star.getIndexInSqlString() == null) { + continue; + } result.put(star.getIndexInSqlString(), star.toSqlWithBacktick()); } } else { List namedExpressions = project.getExcepts(); BoundStar star = project.getBoundStars().get(0); + if (star.getIndexInSqlString() == null) { + return; + } result.put(star.getIndexInSqlString(), star.toSqlWithBacktick(namedExpressions)); } + } else if (node instanceof LogicalCatalogRelation) { + LogicalCatalogRelation logicalCatalogRelation = (LogicalCatalogRelation) node; + if (logicalCatalogRelation.getIndexInSqlString() != null) { + result.put(logicalCatalogRelation.getIndexInSqlString(), + Utils.qualifiedNameWithBacktick(logicalCatalogRelation.qualified())); + } + } else if (node instanceof LogicalSubQueryAlias) { + LogicalSubQueryAlias alias = (LogicalSubQueryAlias) node; + if (alias.getIndexInSqlString() != null) { + result.put(alias.getIndexInSqlString(), + Utils.qualifiedNameWithBacktick(alias.getQualifier())); + } } }); return result; @@ -344,13 +344,9 @@ public List getNamedExpressionContexts() { private static class AnalyzerForCreateView extends AbstractBatchJobExecutor { private final List jobs; - public AnalyzerForCreateView(CascadesContext cascadesContext, boolean forRelation) { + public AnalyzerForCreateView(CascadesContext cascadesContext) { super(cascadesContext); - if (forRelation) { - jobs = buildAnalyzeViewJobsForRelation(); - } else { - jobs = buildAnalyzeViewJobsForStar(); - } + jobs = buildAnalyzeViewJobsForStar(); } public void analyze() { @@ -362,18 +358,10 @@ public List getJobs() { return jobs; } - private static List buildAnalyzeViewJobsForRelation() { - return jobs( - topDown(new AnalyzeCTE()), - topDown(new EliminateLogicalSelectHint()), - bottomUp( - new BindRelationForCreateView() - ) - ); - } - private static List buildAnalyzeViewJobsForStar() { return jobs( + topDown(new EliminateLogicalSelectHint()), + topDown(new AnalyzeCTE()), bottomUp( new BindRelation(), new CheckPolicy(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index 5f7982aae46cab..d62b1355dfd6d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.constraint.PrimaryKeyConstraint; import org.apache.doris.catalog.constraint.UniqueConstraint; +import org.apache.doris.common.Pair; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; @@ -59,6 +60,7 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements protected final TableIf table; // [catalogName, databaseName] protected final ImmutableList qualifier; + private Pair indexInSqlString; public LogicalCatalogRelation(RelationId relationId, PlanType type, TableIf table, List qualifier) { super(relationId, type); @@ -206,4 +208,12 @@ private ImmutableSet findSlotsByColumn(Set outputSet, Set getIndexInSqlString() { + return indexInSqlString; + } + + public void setIndexInSqlString(Pair indexInSqlString) { + this.indexInSqlString = indexInSqlString; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index 68a451f1df1b19..eb60573871f4db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; @@ -54,6 +55,8 @@ public class LogicalSubQueryAlias extends LogicalUnary< private final List qualifier; private final Optional> columnAliases; + private Pair indexInSqlString; + public LogicalSubQueryAlias(String tableAlias, CHILD_TYPE child) { this(ImmutableList.of(tableAlias), Optional.empty(), Optional.empty(), Optional.empty(), child); } @@ -203,4 +206,12 @@ public Set getInputRelations() { relationIdSet.add(relationId); return relationIdSet; } + + public Pair getIndexInSqlString() { + return indexInSqlString; + } + + public void setIndexInSqlString(Pair indexInSqlString) { + this.indexInSqlString = indexInSqlString; + } } diff --git a/regression-test/data/ddl_p0/test_create_view_nereids.out b/regression-test/data/ddl_p0/test_create_view_nereids.out index a7795e15e1150d..52f5bd5747d0ce 100644 --- a/regression-test/data/ddl_p0/test_create_view_nereids.out +++ b/regression-test/data/ddl_p0/test_create_view_nereids.out @@ -47,7 +47,7 @@ v1 CREATE VIEW `v1` COMMENT 'VIEW' AS SELECT\n error_code, \n 1, \n 6 7 1 -- !test_with_as_sql -- -test_view_with_as CREATE VIEW `test_view_with_as` COMMENT 'VIEW' AS (\n with t1 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view), t2 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view), \n t3 as (select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from mal_test_view) SELECT `t1`.`pk`, `t1`.`a`, `t1`.`b` FROM t1); utf8mb4 utf8mb4_0900_bin +test_view_with_as CREATE VIEW `test_view_with_as` COMMENT 'VIEW' AS (\n with t1 as (select `internal`.`regression_test_ddl_p0`.`mal_test_view`.`pk`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`a`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`b` from `internal`.`regression_test_ddl_p0`.`mal_test_view`), t2 as (select `internal`.`regression_test_ddl_p0`.`mal_test_view`.`pk`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`a`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`b` from `internal`.`regression_test_ddl_p0`.`mal_test_view`), \n t3 as (select `internal`.`regression_test_ddl_p0`.`mal_test_view`.`pk`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`a`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`b` from `internal`.`regression_test_ddl_p0`.`mal_test_view`) SELECT `t1`.`pk`, `t1`.`a`, `t1`.`b` FROM t1); utf8mb4 utf8mb4_0900_bin -- !test_union -- 1 1 2 @@ -86,7 +86,7 @@ test_view_with_as CREATE VIEW `test_view_with_as` COMMENT 'VIEW' AS (\n 6 7 1 -- !test_union_sql -- -test_view_union CREATE VIEW `test_view_union` COMMENT 'VIEW' AS (select `regression_test_ddl_p0`.`mal_test_view`.`pk` AS `c1`, `regression_test_ddl_p0`.`mal_test_view`.`a` AS `c2`, `regression_test_ddl_p0`.`mal_test_view`.`b` AS `c3` from `internal`.`regression_test_ddl_p0`.`mal_test_view` Union all SELECT `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` FROM `internal`.`regression_test_ddl_p0`.`mal_test_view`); utf8mb4 utf8mb4_0900_bin +test_view_union CREATE VIEW `test_view_union` COMMENT 'VIEW' AS (select `internal`.`regression_test_ddl_p0`.`mal_test_view`.`pk` AS `c1`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`a` AS `c2`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`b` AS `c3` from `internal`.`regression_test_ddl_p0`.`mal_test_view` Union all SELECT `internal`.`regression_test_ddl_p0`.`mal_test_view`.`pk`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`a`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`b` FROM `internal`.`regression_test_ddl_p0`.`mal_test_view`); utf8mb4 utf8mb4_0900_bin -- !test_count_star -- 17 @@ -139,24 +139,46 @@ test_view_expression CREATE VIEW `test_view_expression` COMMENT 'VIEW' AS (selec test_view_alias CREATE VIEW `test_view_alias` COMMENT 'VIEW' AS (\n select c8 AS `c1`, c2 AS `c2`, c1 AS `c3` from (select a+1 c8,abs(a)+2+1 as c2, cast(b as varchar(10)) as c1 from `internal`.`regression_test_ddl_p0`.`mal_test_view`) t); utf8mb4 utf8mb4_0900_bin -- !test_star_except -- -6 \N 6 +\N 6 +1 2 +1 3 +1 4 +1 4 +1 7 +2 8 +3 5 +3 6 +3 9 +4 2 +5 \N +5 6 +5 6 +5 6 +5 8 +7 1 + +-- !test_star_except_sql -- +test_view_star_except CREATE VIEW `test_view_star_except` COMMENT 'VIEW' AS select `internal`.`regression_test_ddl_p0`.`mal_test_view`.`a`, `internal`.`regression_test_ddl_p0`.`mal_test_view`.`b` from `internal`.`regression_test_ddl_p0`.`mal_test_view`; utf8mb4 utf8mb4_0900_bin + +-- !test_create_view_from_view -- 1 1 2 -2 1 3 1 1 4 +1 3 6 +2 1 3 2 1 4 2 1 7 -3 2 8 2 3 5 -1 3 6 2 3 9 2 4 2 +3 2 8 3 5 \N 3 5 6 3 5 6 -4 5 6 3 5 8 +4 5 6 +6 \N 6 6 7 1 --- !test_star_except_sql -- -test_view_star_except CREATE VIEW `test_view_star_except` COMMENT 'VIEW' AS select `regression_test_ddl_p0`.`mal_test_view`.`pk`, `regression_test_ddl_p0`.`mal_test_view`.`a`, `regression_test_ddl_p0`.`mal_test_view`.`b` from `internal`.`regression_test_ddl_p0`.`mal_test_view`; utf8mb4 utf8mb4_0900_bin +-- !test_create_view_from_view_sql -- +test_view_from_view CREATE VIEW `test_view_from_view` COMMENT 'VIEW' AS select `internal`.`regression_test_ddl_p0`.`test_view_with_as`.`pk` AS `c1`, `internal`.`regression_test_ddl_p0`.`test_view_with_as`.`a` AS `c2`, `internal`.`regression_test_ddl_p0`.`test_view_with_as`.`b` AS `c3` from `internal`.`regression_test_ddl_p0`.`test_view_with_as`; utf8mb4 utf8mb4_0900_bin diff --git a/regression-test/suites/ddl_p0/test_create_view_nereids.groovy b/regression-test/suites/ddl_p0/test_create_view_nereids.groovy index 33aa19239dc2f1..00c56076650d4b 100644 --- a/regression-test/suites/ddl_p0/test_create_view_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_view_nereids.groovy @@ -260,4 +260,10 @@ suite("test_create_view_nereids") { qt_test_star_except "select * from test_view_star_except order by a, b;" qt_test_star_except_sql "show create view test_view_star_except;" + // test create view from view + sql "drop view if exists test_view_from_view" + sql "create view test_view_from_view(c1,c2,c3) as select * from test_view_with_as" + qt_test_create_view_from_view "select * from test_view_from_view order by c1,c2,c3" + qt_test_create_view_from_view_sql "show create view test_view_from_view" + } From 1f6a10eaaea3f81a7c269aaa78d710d6fd23c4a3 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 2 Apr 2024 11:41:27 +0800 Subject: [PATCH 18/23] [feat](nereids) support create view in nereids --- .../apache/doris/analysis/BaseViewStmt.java | 1 - .../apache/doris/analysis/CreateViewStmt.java | 18 --- .../nereids/analyzer/UnboundRelation.java | 8 -- .../properties/UnboundLogicalProperties.java | 2 +- .../apache/doris/nereids/rules/RuleType.java | 2 - .../analysis/BindRelationForCreateView.java | 48 ------- .../doris/nereids/trees/plans/FakePlan.java | 2 +- .../org/apache/doris/nereids/util/Utils.java | 9 +- ...st_show_create_table_and_views_nereids.out | 46 +++++++ .../test_show_create_table_and_views.groovy | 2 + ...show_create_table_and_views_nereids.groovy | 128 ++++++++++++++++++ 11 files changed, 181 insertions(+), 85 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java create mode 100644 regression-test/data/show_p0/test_show_create_table_and_views_nereids.out create mode 100644 regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseViewStmt.java index 468a3de80245a9..d8740f03f523b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseViewStmt.java @@ -49,7 +49,6 @@ public class BaseViewStmt extends DdlStmt { protected QueryStmt cloneStmt; public BaseViewStmt(TableName tableName, List cols, QueryStmt queryStmt) { - // Preconditions.checkNotNull(queryStmt); this.tableName = tableName; this.cols = cols; this.viewDefStmt = queryStmt; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java index b85fb50245422c..2eed97f0a018e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java @@ -25,8 +25,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; @@ -96,22 +94,6 @@ public void analyze(Analyzer analyzer) throws UserException { } } - public void createColumnAndViewDefsForNereids(Plan plan) { - List outputs = plan.getOutput(); - if (cols.isEmpty()) { - for (Slot output : outputs) { - Column column = new Column(output.getName(), output.getDataType().toCatalogDataType()); - finalCols.add(column); - } - } else { - for (int i = 0; i < cols.size(); ++i) { - Column column = new Column(cols.get(i).getColName(), outputs.get(i).getDataType().toCatalogDataType()); - column.setComment(cols.get(i).getComment()); - finalCols.add(column); - } - } - } - public void setInlineViewDef(String querySql) { inlineViewDef = querySql; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 485cdc6c1290c6..0a6bd2ff6999f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -202,12 +202,4 @@ public TableScanParams getScanParams() { public Pair getIndexInSqlString() { return indexInSqlString; } - - public List getTableQualifier() { - return tableQualifier; - } - - public void setTableQualifier(List tableQualifier) { - this.tableQualifier = tableQualifier; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java index eaca9ef7769177..886b7aceb13dda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/UnboundLogicalProperties.java @@ -32,7 +32,7 @@ public class UnboundLogicalProperties extends LogicalProperties { public static final UnboundLogicalProperties INSTANCE = new UnboundLogicalProperties(); private UnboundLogicalProperties() { - super(ImmutableList::of, () -> FunctionalDependencies.EMPTY_FUNC_DEPS, ImmutableList::of); + super(ImmutableList::of, () -> FunctionalDependencies.EMPTY_FUNC_DEPS); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index db34bb32c7566a..32c026d3c63afa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -420,8 +420,6 @@ public enum RuleType { LOGICAL_GENERATE_TO_PHYSICAL_GENERATE(RuleTypeClass.IMPLEMENTATION), LOGICAL_WINDOW_TO_PHYSICAL_WINDOW_RULE(RuleTypeClass.IMPLEMENTATION), IMPLEMENTATION_SENTINEL(RuleTypeClass.IMPLEMENTATION), - BINDING_RELATION_FOR_CREATE_VIEW(RuleTypeClass.REWRITE), - // sentinel, use to count rules SENTINEL(RuleTypeClass.SENTINEL), ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java deleted file mode 100644 index a29d8754dfd0b9..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelationForCreateView.java +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.analysis; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.analyzer.UnboundRelation; -import org.apache.doris.nereids.pattern.MatchingContext; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.util.RelationUtil; - -import java.util.List; - -/** BindRelationForCreateView */ -public class BindRelationForCreateView extends OneAnalysisRuleFactory { - @Override - public Rule build() { - return unboundRelation().thenApply(this::doBindRelation).toRule(RuleType.BINDING_RELATION_FOR_CREATE_VIEW); - } - - private Plan doBindRelation(MatchingContext ctx) { - return bind(ctx.cascadesContext, ctx.root); - } - - private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboundRelation) { - List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), - unboundRelation.getNameParts()); - unboundRelation.setTableQualifier(tableQualifier); - return unboundRelation; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java index eedfbbd8aa2ad2..ad186c680ce08c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/FakePlan.java @@ -78,7 +78,7 @@ public List getExpressions() { @Override public LogicalProperties getLogicalProperties() { - return new LogicalProperties(ArrayList::new, () -> FunctionalDependencies.EMPTY_FUNC_DEPS, ArrayList::new); + return new LogicalProperties(ArrayList::new, () -> FunctionalDependencies.EMPTY_FUNC_DEPS); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java index a6a992ef0e3513..5888fc9662f9bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java @@ -130,12 +130,9 @@ public static String qualifiedName(List qualifier, String name) { /** get qualified name with Backtick */ public static String qualifiedNameWithBacktick(List qualifiers, String name) { - List qualifierWithBacktick = Lists.newArrayListWithCapacity(qualifiers.size()); - for (String qualifier : qualifiers) { - qualifierWithBacktick.add('`' + qualifier + '`'); - } - String nameWithBacktick = '`' + name + '`'; - return StringUtils.join(qualifiedNameParts(qualifierWithBacktick, nameWithBacktick), "."); + List fullName = new ArrayList<>(qualifiers); + fullName.add(name); + return qualifiedNameWithBacktick(fullName); } public static String qualifiedNameWithBacktick(List qualifiers) { diff --git a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out new file mode 100644 index 00000000000000..3a5728d3d9e50a --- /dev/null +++ b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out @@ -0,0 +1,46 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !show -- +show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !select -- +1 1 30 +1 2 5 +1 3 10 +1 300 2 +2 1 100 +2 2 10 +2 3 44 +2 200 1111 +3 1 10 +3 2 1 +23 900 1 +100 100 1 +200 20 1 +300 20 1 + +-- !select -- +1 5 +2 10 +3 1 + +-- !show -- +show_create_table_and_views_nereids_view CREATE VIEW `show_create_table_and_views_nereids_view` COMMENT 'VIEW' AS SELECT user_id AS `user_id`, cost AS `cost` FROM `internal`.`show_create_table_and_views_nereids_db`.`show_create_table_and_views_nereids_table`\n WHERE good_id = 2; utf8mb4 utf8mb4_0900_bin + +-- !select -- +1 47 +2 1265 +3 11 +23 1 +100 1 +200 1 +300 1 + +-- !show -- +show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !show -- +show_create_table_and_views_nereids_like CREATE TABLE `show_create_table_and_views_nereids_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !show -- +show_create_table_and_views_nereids_like_with_rollup CREATE TABLE `show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + diff --git a/regression-test/suites/show_p0/test_show_create_table_and_views.groovy b/regression-test/suites/show_p0/test_show_create_table_and_views.groovy index 6c6322e5c4aa03..5872cc95d5ae5c 100644 --- a/regression-test/suites/show_p0/test_show_create_table_and_views.groovy +++ b/regression-test/suites/show_p0/test_show_create_table_and_views.groovy @@ -30,6 +30,7 @@ suite("test_show_create_table_and_views", "show") { String rollupName = "${suiteName}_rollup" String likeName = "${suiteName}_like" + sql "SET enable_nereids_planner=false;" sql "CREATE DATABASE IF NOT EXISTS ${dbName}" sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" sql """ @@ -76,6 +77,7 @@ suite("test_show_create_table_and_views", "show") { qt_show "SHOW CREATE TABLE ${dbName}.${tableName}" qt_select "SELECT * FROM ${dbName}.${tableName} ORDER BY user_id, good_id" + sql "drop view if exists ${dbName}.${viewName};" // create view and show sql """ CREATE VIEW IF NOT EXISTS ${dbName}.${viewName} (user_id, cost) diff --git a/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy new file mode 100644 index 00000000000000..88fd00444adec6 --- /dev/null +++ b/regression-test/suites/show_p0/test_show_create_table_and_views_nereids.groovy @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_show_create_table_and_views_nereids", "show") { + def ret = sql "SHOW FRONTEND CONFIG like '%enable_feature_binlog%';" + logger.info("${ret}") + if (ret.size() != 0 && ret[0].size() > 1 && ret[0][1] == 'false') { + logger.info("enable_feature_binlog=false in frontend config, no need to run this case.") + return + } + + String suiteName = "show_create_table_and_views_nereids" + String dbName = "${suiteName}_db" + String tableName = "${suiteName}_table" + String viewName = "${suiteName}_view" + String rollupName = "${suiteName}_rollup" + String likeName = "${suiteName}_like" + + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `user_id` LARGEINT NOT NULL, + `good_id` LARGEINT NOT NULL, + `cost` BIGINT SUM DEFAULT "0" + ) + AGGREGATE KEY(`user_id`, `good_id`) + PARTITION BY RANGE(`good_id`) + ( + PARTITION p1 VALUES LESS THAN ("100"), + PARTITION p2 VALUES LESS THAN ("200"), + PARTITION p3 VALUES LESS THAN ("300"), + PARTITION p4 VALUES LESS THAN ("400"), + PARTITION p5 VALUES LESS THAN ("500"), + PARTITION p6 VALUES LESS THAN ("600"), + PARTITION p7 VALUES LESS THAN MAXVALUE + ) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """INSERT INTO ${dbName}.${tableName} VALUES + (1, 1, 10), + (1, 1, 20), + (1, 2, 5), + (1, 3, 10), + (2, 1, 0), + (2, 1, 100), + (3, 1, 10), + (2, 2, 10), + (2, 3, 44), + (3, 2, 1), + (100, 100, 1), + (200, 20, 1), + (300, 20, 1), + (1, 300, 2), + (2, 200, 1111), + (23, 900, 1)""" + + qt_show "SHOW CREATE TABLE ${dbName}.${tableName}" + qt_select "SELECT * FROM ${dbName}.${tableName} ORDER BY user_id, good_id" + + sql "drop view if exists ${dbName}.${viewName};" + // create view and show + sql """ + CREATE VIEW IF NOT EXISTS ${dbName}.${viewName} (user_id, cost) + AS + SELECT user_id, cost FROM ${dbName}.${tableName} + WHERE good_id = 2 + """ + qt_select "SELECT * FROM ${dbName}.${viewName} ORDER BY user_id" + qt_show "SHOW CREATE VIEW ${dbName}.${viewName}" + + // create rollup + sql """ALTER TABLE ${dbName}.${tableName} + ADD ROLLUP ${rollupName} (user_id, cost) + """ + + def isAlterTableFinish = { -> + def records = sql """SHOW ALTER TABLE ROLLUP FROM ${dbName}""" + for (def row in records) { + if (row[5] == "${rollupName}" && row[8] == "FINISHED") { + return true + } + } + false + } + while (!isAlterTableFinish()) { + Thread.sleep(100) + } + + qt_select "SELECT user_id, SUM(cost) FROM ${dbName}.${tableName} GROUP BY user_id ORDER BY user_id" + qt_show "SHOW CREATE TABLE ${dbName}.${tableName}" + + // create like + sql "CREATE TABLE ${dbName}.${likeName} LIKE ${dbName}.${tableName}" + qt_show "SHOW CREATE TABLE ${dbName}.${likeName}" + + // create like with rollup + sql "CREATE TABLE ${dbName}.${likeName}_with_rollup LIKE ${dbName}.${tableName} WITH ROLLUP" + qt_show "SHOW CREATE TABLE ${dbName}.${likeName}_with_rollup" + + sql "DROP TABLE IF EXISTS ${dbName}.${likeName}_with_rollup FORCE" + sql "DROP TABLE ${dbName}.${likeName} FORCE" + sql "DROP VIEW ${dbName}.${viewName}" + sql "DROP TABLE ${dbName}.${tableName} FORCE" + sql "DROP DATABASE ${dbName} FORCE" +} + From 2441c00018d806104cde508214dc06d5bf220077 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 3 Apr 2024 18:10:11 +0800 Subject: [PATCH 19/23] [feat](nereids) support create view in nereids --- .../doris/nereids/StatementContext.java | 14 ++++ .../nereids/analyzer/UnboundRelation.java | 2 +- .../doris/nereids/analyzer/UnboundStar.java | 2 +- .../rules/analysis/BindExpression.java | 17 +++-- .../nereids/rules/analysis/BindRelation.java | 65 +++++++------------ .../rules/analysis/ExpressionAnalyzer.java | 10 ++- .../nereids/trees/expressions/BoundStar.java | 40 ------------ .../plans/commands/info/CreateViewInfo.java | 48 +------------- .../plans/logical/LogicalCatalogRelation.java | 10 --- .../trees/plans/logical/LogicalProject.java | 10 --- .../plans/logical/LogicalSubQueryAlias.java | 11 ---- 11 files changed, 55 insertions(+), 174 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 7b444995120cab..403c605ba75992 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -54,6 +54,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import javax.annotation.concurrent.GuardedBy; /** @@ -121,6 +122,11 @@ public class StatementContext { private BitSet disableRules; + // for create view support in nereids + // key is the start and end position of the sql substring that needs to be replaced, + // and value is the new string used for replacement. + private TreeMap, String> indexInSqlToString = new TreeMap<>(new Pair.PairComparator<>()); + public StatementContext() { this.connectContext = ConnectContext.get(); } @@ -354,4 +360,12 @@ public boolean isHasUnknownColStats() { public void setHasUnknownColStats(boolean hasUnknownColStats) { this.hasUnknownColStats = hasUnknownColStats; } + + public TreeMap, String> getIndexInSqlToString() { + return indexInSqlToString; + } + + public void addIndexInSqlToString(Pair pair, String replacement) { + indexInSqlToString.put(pair, replacement); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 0a6bd2ff6999f9..30e57111cbd898 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -56,8 +56,8 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu private final Optional tableSample; private final Optional indexName; private TableScanParams scanParams; + // the start and end position of the sql substring(e.g. "t1", "db1.t1", "ctl1.db1.t1") private Pair indexInSqlString; - private List tableQualifier; public UnboundRelation(RelationId id, List nameParts) { this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java index 14cfaffeb47fea..1d37b12250ad18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java @@ -35,7 +35,7 @@ */ public class UnboundStar extends NamedExpression implements LeafExpression, Unbound, PropagateNullable { private final List qualifier; - + // the start and end position of the sql substring(e.g. "*", "table.*") private Pair indexInSqlString; public UnboundStar(List qualifier) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 9b12b01917c0be..2fe2d36c52fd01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.analyzer.UnboundStar; import org.apache.doris.nereids.analyzer.UnboundTVFRelation; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.LogicalPlanBuilder; @@ -568,24 +569,25 @@ private Plan bindProject(MatchingContext> ctx) { () -> analyzer.analyzeToSet(project.getExcepts())); Builder boundProjections = ImmutableList.builderWithExpectedSize(project.arity()); - List boundStars = Lists.newArrayList(); + StatementContext statementContext = ctx.statementContext; for (Expression expression : project.getProjects()) { Expression expr = analyzer.analyze(expression); if (!(expr instanceof BoundStar)) { boundProjections.add((NamedExpression) expr); } else { BoundStar boundStar = (BoundStar) expr; - boundStars.add(boundStar); List slots = boundStar.getSlots(); if (!excepts.isEmpty()) { slots = Utils.filterImmutableList(slots, slot -> !boundExcepts.get().contains(slot)); } boundProjections.addAll(slots); + UnboundStar unboundStar = (UnboundStar) expression; + if (unboundStar.getIndexInSqlString() != null) { + statementContext.addIndexInSqlToString(unboundStar.getIndexInSqlString(), toSqlWithBacktick(slots)); + } } } - LogicalProject finalProject = project.withProjects(boundProjections.build()); - finalProject.setBoundStars(boundStars); - return finalProject; + return project.withProjects(boundProjections.build()); } private Plan bindFilter(MatchingContext> ctx) { @@ -944,4 +946,9 @@ default Set analyzeToSet(List exprs) { private interface CustomSlotBinderAnalyzer { List bindSlot(ExpressionAnalyzer analyzer, UnboundSlot unboundSlot); } + + public String toSqlWithBacktick(List slots) { + return slots.stream().map(slot -> ((SlotReference) slot).getQualifiedNameWithBacktick()) + .collect(Collectors.joining(", ")); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 7bc09d902cc43a..c6c15fedaf5209 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -31,6 +31,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.nereids.CTEContext; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.Unbound; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundResultSink; @@ -62,6 +63,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan; import org.apache.doris.nereids.trees.plans.logical.LogicalView; import org.apache.doris.nereids.util.RelationUtil; +import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; @@ -215,7 +217,6 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, unboundRelation.getTableSample()); } } - scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign() && !ConnectContext.get().getSessionVariable().skipDeleteSign() && !scan.isDirectMvScan()) { @@ -240,72 +241,50 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation, List tableQualifier, CascadesContext cascadesContext) { + if (unboundRelation.getIndexInSqlString() != null) { + StatementContext statementContext = cascadesContext.getStatementContext(); + statementContext.addIndexInSqlToString(unboundRelation.getIndexInSqlString(), + Utils.qualifiedNameWithBacktick(tableQualifier)); + } List qualifierWithoutTableName = Lists.newArrayList(); qualifierWithoutTableName.addAll(tableQualifier.subList(0, tableQualifier.size() - 1)); - switch (table.getType()) { case OLAP: case MATERIALIZED_VIEW: return makeOlapScan(table, unboundRelation, qualifierWithoutTableName); - case VIEW: { + case VIEW: View view = (View) table; String inlineViewDef = view.getInlineViewDef(); Plan viewBody = parseAndAnalyzeView(inlineViewDef, cascadesContext); LogicalView logicalView = new LogicalView<>(view, viewBody); - LogicalSubQueryAlias alias = new LogicalSubQueryAlias<>(tableQualifier, logicalView); - alias.setIndexInSqlString(unboundRelation.getIndexInSqlString()); - return alias; - } - case HMS_EXTERNAL_TABLE: { + return new LogicalSubQueryAlias<>(tableQualifier, logicalView); + case HMS_EXTERNAL_TABLE: HMSExternalTable hmsTable = (HMSExternalTable) table; if (Config.enable_query_hive_views && hmsTable.isView()) { String hiveCatalog = hmsTable.getCatalog().getName(); String ddlSql = hmsTable.getViewText(); Plan hiveViewPlan = parseAndAnalyzeHiveView(hiveCatalog, ddlSql, cascadesContext); - LogicalSubQueryAlias alias = new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan); - alias.setIndexInSqlString(unboundRelation.getIndexInSqlString()); - return alias; + return new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan); } hmsTable.setScanParams(unboundRelation.getScanParams()); - LogicalFileScan scan = new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, + return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, qualifierWithoutTableName, unboundRelation.getTableSample()); - scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); - return scan; - } case ICEBERG_EXTERNAL_TABLE: case PAIMON_EXTERNAL_TABLE: case MAX_COMPUTE_EXTERNAL_TABLE: - case TRINO_CONNECTOR_EXTERNAL_TABLE: { - LogicalFileScan scan = new LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table, + case TRINO_CONNECTOR_EXTERNAL_TABLE: + return new LogicalFileScan(unboundRelation.getRelationId(), (ExternalTable) table, qualifierWithoutTableName, unboundRelation.getTableSample()); - scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); - return scan; - } - case SCHEMA: { - LogicalSchemaScan scan = new LogicalSchemaScan(unboundRelation.getRelationId(), table, - qualifierWithoutTableName); - scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); - return scan; - } + case SCHEMA: + return new LogicalSchemaScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); case JDBC_EXTERNAL_TABLE: - case JDBC: { - LogicalJdbcScan scan = new LogicalJdbcScan(unboundRelation.getRelationId(), table, - qualifierWithoutTableName); - scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); - return scan; - } - case ODBC: { - LogicalOdbcScan scan = new LogicalOdbcScan(unboundRelation.getRelationId(), table, + case JDBC: + return new LogicalJdbcScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); + case ODBC: + return new LogicalOdbcScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); + case ES_EXTERNAL_TABLE: + return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, qualifierWithoutTableName); - scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); - return scan; - } - case ES_EXTERNAL_TABLE: { - LogicalEsScan scan = new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, - qualifierWithoutTableName); - scan.setIndexInSqlString(unboundRelation.getIndexInSqlString()); - return scan; - } case TEST_EXTERNAL_TABLE: return new LogicalTestScan(unboundRelation.getRelationId(), table, qualifierWithoutTableName); default: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index 745cc2de9db326..56ca1b3a8c4822 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FunctionRegistry; import org.apache.doris.common.DdlException; -import org.apache.doris.common.Pair; import org.apache.doris.common.util.Util; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -276,11 +275,11 @@ public Expression visitUnboundStar(UnboundStar unboundStar, ExpressionRewriteCon .collect(Collectors.toList()); switch (qualifier.size()) { case 0: // select * - return new BoundStar(slots, unboundStar.getIndexInSqlString()); + return new BoundStar(slots); case 1: // select table.* case 2: // select db.table.* case 3: // select catalog.db.table.* - return bindQualifiedStar(qualifier, slots, unboundStar.getIndexInSqlString()); + return bindQualifiedStar(qualifier, slots); default: throw new AnalysisException("Not supported qualifier: " + StringUtils.join(qualifier, ".")); @@ -576,8 +575,7 @@ public Expression visitCast(Cast cast, ExpressionRewriteContext context) { return cast; } - private BoundStar bindQualifiedStar(List qualifierStar, List boundSlots, - Pair indexInSql) { + private BoundStar bindQualifiedStar(List qualifierStar, List boundSlots) { // FIXME: compatible with previous behavior: // https://github.com/apache/doris/pull/10415/files/3fe9cb0c3f805ab3a9678033b281b16ad93ec60a#r910239452 List slots = boundSlots.stream().filter(boundSlot -> { @@ -641,7 +639,7 @@ && compareDbName(qualifierStar.get(1), boundSlotQualifier.get(1)) if (slots.isEmpty()) { throw new AnalysisException("unknown qualifier: " + StringUtils.join(qualifierStar, ".") + ".*"); } - return new BoundStar(slots, indexInSql); + return new BoundStar(slots); } protected List bindSlotByThisScope(UnboundSlot unboundSlot) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java index 781c53ad65f0da..8b4bffad3fc817 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BoundStar.java @@ -17,23 +17,17 @@ package org.apache.doris.nereids.trees.expressions; -import org.apache.doris.common.Pair; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; /** BoundStar is used to wrap list of slots for temporary. */ public class BoundStar extends NamedExpression implements PropagateNullable { - private Pair indexInSqlString; - public BoundStar(List children) { super((List) children); Preconditions.checkArgument(children.stream().noneMatch(slot -> slot instanceof UnboundSlot), @@ -41,49 +35,15 @@ public BoundStar(List children) { ); } - public BoundStar(List children, Pair indexInSqlString) { - super((List) children); - Preconditions.checkArgument(children.stream().noneMatch(slot -> slot instanceof UnboundSlot), - "BoundStar can not wrap UnboundSlot" - ); - this.indexInSqlString = indexInSqlString; - } - public String toSql() { return children.stream().map(Expression::toSql).collect(Collectors.joining(", ")); } - public String toSqlWithBacktick() { - return children.stream().map(slot -> ((SlotReference) slot).getQualifiedNameWithBacktick()) - .collect(Collectors.joining(", ")); - } - - /** toSqlWithBacktick */ - public String toSqlWithBacktick(List excepts) { - List outputs = Lists.newArrayList(); - Set exceptSet = Sets.newHashSet(); - for (NamedExpression except : excepts) { - exceptSet.add(except.getName()); - } - for (Expression child : children) { - SlotReference slot = (SlotReference) child; - if (!exceptSet.contains(slot.getName())) { - outputs.add(child); - } - } - return outputs.stream().map(slot -> ((SlotReference) slot).getQualifiedNameWithBacktick()) - .collect(Collectors.joining(", ")); - } - @SuppressWarnings({"unchecked", "rawtypes"}) public List getSlots() { return (List) children(); } - public Pair getIndexInSqlString() { - return indexInSqlString; - } - @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitBoundStar(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index f431ce6b8fe353..1d0bee75fb8178 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -46,20 +46,14 @@ import org.apache.doris.nereids.rules.analysis.BindRelation; import org.apache.doris.nereids.rules.analysis.CheckPolicy; import org.apache.doris.nereids.rules.analysis.EliminateLogicalSelectHint; -import org.apache.doris.nereids.trees.expressions.BoundStar; -import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; -import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; -import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Lists; @@ -72,7 +66,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; /** @@ -88,7 +81,6 @@ public class CreateViewInfo { private final List finalCols = Lists.newArrayList(); private final List outputs = Lists.newArrayList(); private Plan analyzedPlan; - private Map, String> rewriteSqlMap; /** constructor*/ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String comment, LogicalPlan logicalQuery, @@ -144,7 +136,7 @@ public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { CreateViewStmt createViewStmt = new CreateViewStmt(ifNotExists, viewName.transferToTableName(), cols, comment, null); // expand star(*) in project list and replace table name with qualifier - String rewrittenSql = rewriteSql(rewriteSqlMap); + String rewrittenSql = rewriteSql(ctx.getStatementContext().getIndexInSqlToString()); // rewrite project alias rewrittenSql = rewriteProjectsToUserDefineAlias(rewrittenSql); @@ -165,7 +157,6 @@ private void analyzeAndFillRewriteSqlMap(String sql, ConnectContext ctx) { AnalyzerForCreateView analyzerForStar = new AnalyzerForCreateView(viewContextForStar); analyzerForStar.analyze(); analyzedPlan = viewContextForStar.getRewritePlan(); - rewriteSqlMap = collectStarAndRelationInfo(analyzedPlan); } private String rewriteSql(Map, String> indexStringSqlMap) { @@ -227,43 +218,6 @@ private void createFinalCols() throws org.apache.doris.common.AnalysisException } } - private Map, String> collectStarAndRelationInfo(Plan plan) { - TreeMap, String> result = new TreeMap<>(new Pair.PairComparator<>()); - plan.foreach(node -> { - if (node instanceof LogicalProject) { - LogicalProject project = (LogicalProject) node; - if (project.getExcepts().isEmpty()) { - for (BoundStar star : project.getBoundStars()) { - if (star.getIndexInSqlString() == null) { - continue; - } - result.put(star.getIndexInSqlString(), star.toSqlWithBacktick()); - } - } else { - List namedExpressions = project.getExcepts(); - BoundStar star = project.getBoundStars().get(0); - if (star.getIndexInSqlString() == null) { - return; - } - result.put(star.getIndexInSqlString(), star.toSqlWithBacktick(namedExpressions)); - } - } else if (node instanceof LogicalCatalogRelation) { - LogicalCatalogRelation logicalCatalogRelation = (LogicalCatalogRelation) node; - if (logicalCatalogRelation.getIndexInSqlString() != null) { - result.put(logicalCatalogRelation.getIndexInSqlString(), - Utils.qualifiedNameWithBacktick(logicalCatalogRelation.qualified())); - } - } else if (node instanceof LogicalSubQueryAlias) { - LogicalSubQueryAlias alias = (LogicalSubQueryAlias) node; - if (alias.getIndexInSqlString() != null) { - result.put(alias.getIndexInSqlString(), - Utils.qualifiedNameWithBacktick(alias.getQualifier())); - } - } - }); - return result; - } - private static class OutermostPlanFinder extends DefaultPlanVisitor> { boolean found = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java index d62b1355dfd6d9..5f7982aae46cab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCatalogRelation.java @@ -24,7 +24,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.constraint.PrimaryKeyConstraint; import org.apache.doris.catalog.constraint.UniqueConstraint; -import org.apache.doris.common.Pair; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; @@ -60,7 +59,6 @@ public abstract class LogicalCatalogRelation extends LogicalRelation implements protected final TableIf table; // [catalogName, databaseName] protected final ImmutableList qualifier; - private Pair indexInSqlString; public LogicalCatalogRelation(RelationId relationId, PlanType type, TableIf table, List qualifier) { super(relationId, type); @@ -208,12 +206,4 @@ private ImmutableSet findSlotsByColumn(Set outputSet, Set getIndexInSqlString() { - return indexInSqlString; - } - - public void setIndexInSqlString(Pair indexInSqlString) { - this.indexInSqlString = indexInSqlString; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index e13def90b3f4a6..89dd7d49677d3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -40,7 +40,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; import org.json.JSONObject; import java.util.HashSet; @@ -59,7 +58,6 @@ public class LogicalProject extends LogicalUnary excepts; private final boolean isDistinct; private final boolean canEliminate; - private List boundStars = Lists.newArrayList(); public LogicalProject(List projects, CHILD_TYPE child) { this(projects, ImmutableList.of(), false, true, ImmutableList.of(child)); @@ -273,12 +271,4 @@ public ImmutableSet computeFdItems(Supplier> outputSupplier) return builder.build(); } - - public List getBoundStars() { - return boundStars; - } - - public void setBoundStars(List boundStars) { - this.boundStars = boundStars; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index eb60573871f4db..68a451f1df1b19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.trees.plans.logical; -import org.apache.doris.common.Pair; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.FdItem; import org.apache.doris.nereids.properties.FunctionalDependencies; @@ -55,8 +54,6 @@ public class LogicalSubQueryAlias extends LogicalUnary< private final List qualifier; private final Optional> columnAliases; - private Pair indexInSqlString; - public LogicalSubQueryAlias(String tableAlias, CHILD_TYPE child) { this(ImmutableList.of(tableAlias), Optional.empty(), Optional.empty(), Optional.empty(), child); } @@ -206,12 +203,4 @@ public Set getInputRelations() { relationIdSet.add(relationId); return relationIdSet; } - - public Pair getIndexInSqlString() { - return indexInSqlString; - } - - public void setIndexInSqlString(Pair indexInSqlString) { - this.indexInSqlString = indexInSqlString; - } } From 2d75e0815845a532ab34193427b316e31084b104 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 3 Apr 2024 18:21:26 +0800 Subject: [PATCH 20/23] [feat](nereids) support create view in nereids --- .../org/apache/doris/nereids/rules/analysis/BindExpression.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index df29b60fe8d62d..11ff737f9d55d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -983,6 +983,7 @@ private interface CustomSlotBinderAnalyzer { public String toSqlWithBacktick(List slots) { return slots.stream().map(slot -> ((SlotReference) slot).getQualifiedNameWithBacktick()) .collect(Collectors.joining(", ")); + } private boolean hasAggregateFunction(Expression expression, FunctionRegistry functionRegistry) { return expression.anyMatch(expr -> { From 65aaab769c4eaa86babd298c2cfcd9ae9438806a Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 8 Apr 2024 12:29:13 +0800 Subject: [PATCH 21/23] [feat](nereids) support create view in nereids --- .../nereids/analyzer/UnboundRelation.java | 18 ++++---- .../doris/nereids/analyzer/UnboundStar.java | 8 ++-- .../nereids/parser/LogicalPlanBuilder.java | 8 ++-- .../apache/doris/nereids/rules/RuleType.java | 1 + .../rules/analysis/BindExpression.java | 13 +++--- .../nereids/rules/analysis/BindRelation.java | 9 ++-- .../trees/expressions/SlotReference.java | 4 +- .../plans/commands/info/CreateViewInfo.java | 9 ++-- .../commands/info/SimpleColumnDefinition.java | 2 +- .../org/apache/doris/nereids/util/Utils.java | 14 +++--- .../data/ddl_p0/test_create_view_nereids.out | 44 +++++++++++++++++++ .../ddl_p0/test_create_view_nereids.groovy | 11 +++++ 12 files changed, 102 insertions(+), 39 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 30e57111cbd898..34217b249fc510 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -57,41 +57,41 @@ public class UnboundRelation extends LogicalRelation implements Unbound, BlockFu private final Optional indexName; private TableScanParams scanParams; // the start and end position of the sql substring(e.g. "t1", "db1.t1", "ctl1.db1.t1") - private Pair indexInSqlString; + private final Optional> indexInSqlString; public UnboundRelation(RelationId id, List nameParts) { this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of(), - ImmutableList.of(), Optional.empty(), Optional.empty(), null, null); + ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart) { this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of(), - ImmutableList.of(), Optional.empty(), Optional.empty(), null, null); + ImmutableList.of(), Optional.empty(), Optional.empty(), null, Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, null); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, TableScanParams scanParams) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, null); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName) { this(id, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample, indexName, null, null); + isTempPart, tabletIds, hints, tableSample, indexName, null, Optional.empty()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, - TableScanParams scanParams, Pair indexInSqlString) { + TableScanParams scanParams, Optional> indexInSqlString) { this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString); } @@ -102,7 +102,7 @@ public UnboundRelation(RelationId id, List nameParts, List partN public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, Optional logicalProperties, List partNames, boolean isTempPart, List tabletIds, List hints, Optional tableSample, Optional indexName, - TableScanParams scanParams, Pair indexInSqlString) { + TableScanParams scanParams, Optional> indexInSqlString) { super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties); this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null")); this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null")); @@ -199,7 +199,7 @@ public TableScanParams getScanParams() { return scanParams; } - public Pair getIndexInSqlString() { + public Optional> getIndexInSqlString() { return indexInSqlString; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java index 1d37b12250ad18..cf73f877d604eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Objects; +import java.util.Optional; /** * Star expression. @@ -36,14 +37,15 @@ public class UnboundStar extends NamedExpression implements LeafExpression, Unbound, PropagateNullable { private final List qualifier; // the start and end position of the sql substring(e.g. "*", "table.*") - private Pair indexInSqlString; + private final Optional> indexInSqlString; public UnboundStar(List qualifier) { super(ImmutableList.of()); this.qualifier = Objects.requireNonNull(ImmutableList.copyOf(qualifier), "qualifier can not be null"); + this.indexInSqlString = Optional.empty(); } - public UnboundStar(List qualifier, Pair indexInSqlString) { + public UnboundStar(List qualifier, Optional> indexInSqlString) { super(ImmutableList.of()); this.qualifier = Objects.requireNonNull(ImmutableList.copyOf(qualifier), "qualifier can not be null"); this.indexInSqlString = indexInSqlString; @@ -79,7 +81,7 @@ public boolean equals(Object o) { return qualifier.equals(that.qualifier); } - public Pair getIndexInSqlString() { + public Optional> getIndexInSqlString() { return indexInSqlString; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 81e364ba0c98b8..c085403ba7b62c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1346,7 +1346,7 @@ public LogicalPlan visitTableName(TableNameContext ctx) { UnboundRelation relation = forCreateView ? new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, Optional.ofNullable(tableSample), indexName, scanParams, - Pair.of(identifier.start.getStartIndex(), identifier.stop.getStopIndex())) : + Optional.of(Pair.of(identifier.start.getStartIndex(), identifier.stop.getStopIndex()))) : new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, Optional.ofNullable(tableSample), indexName, scanParams); @@ -1400,7 +1400,7 @@ public Expression visitStar(StarContext ctx) { target = ImmutableList.of(); } return forCreateView - ? new UnboundStar(target, Pair.of(ctx.start.getStartIndex(), ctx.stop.getStopIndex())) : + ? new UnboundStar(target, Optional.of(Pair.of(ctx.start.getStartIndex(), ctx.stop.getStopIndex()))) : new UnboundStar(target); }); } @@ -2891,7 +2891,7 @@ private LogicalPlan withSelectQuerySpecification( throw new ParseException("only column name is supported in except clause", selectColumnCtx); } UnboundStar star = forCreateView ? new UnboundStar(ImmutableList.of(), - Pair.of(selectColumnCtx.start.getStartIndex(), selectColumnCtx.stop.getStopIndex())) : + Optional.of(Pair.of(selectColumnCtx.start.getStartIndex(), selectColumnCtx.stop.getStopIndex()))) : new UnboundStar(ImmutableList.of()); project = new LogicalProject<>(ImmutableList.of(star), expressions, isDistinct, aggregate); } else { @@ -3073,7 +3073,7 @@ private LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContext throw new ParseException("only column name is supported in except clause", selectCtx); } UnboundStar star = forCreateView ? new UnboundStar(ImmutableList.of(), - Pair.of(selectCtx.start.getStartIndex(), selectCtx.stop.getStopIndex())) : + Optional.of(Pair.of(selectCtx.start.getStartIndex(), selectCtx.stop.getStopIndex()))) : new UnboundStar(ImmutableList.of()); return new LogicalProject<>(ImmutableList.of(star), expressions, isDistinct, input); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index d0b6f985a3a638..3a43e1ad672188 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -421,6 +421,7 @@ public enum RuleType { LOGICAL_GENERATE_TO_PHYSICAL_GENERATE(RuleTypeClass.IMPLEMENTATION), LOGICAL_WINDOW_TO_PHYSICAL_WINDOW_RULE(RuleTypeClass.IMPLEMENTATION), IMPLEMENTATION_SENTINEL(RuleTypeClass.IMPLEMENTATION), + // sentinel, use to count rules SENTINEL(RuleTypeClass.SENTINEL), ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 11ff737f9d55d9..97cd6cda6699fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -583,10 +583,13 @@ private Plan bindProject(MatchingContext> ctx) { slots = Utils.filterImmutableList(slots, slot -> !boundExcepts.get().contains(slot)); } boundProjections.addAll(slots); + + // for create view stmt expand star + List slotsForLambda = slots; UnboundStar unboundStar = (UnboundStar) expression; - if (unboundStar.getIndexInSqlString() != null) { - statementContext.addIndexInSqlToString(unboundStar.getIndexInSqlString(), toSqlWithBacktick(slots)); - } + unboundStar.getIndexInSqlString().ifPresent(pair -> + statementContext.addIndexInSqlToString(pair, toSqlWithBackquote(slotsForLambda)) + ); } } return project.withProjects(boundProjections.build()); @@ -980,8 +983,8 @@ private interface CustomSlotBinderAnalyzer { List bindSlot(ExpressionAnalyzer analyzer, UnboundSlot unboundSlot); } - public String toSqlWithBacktick(List slots) { - return slots.stream().map(slot -> ((SlotReference) slot).getQualifiedNameWithBacktick()) + public String toSqlWithBackquote(List slots) { + return slots.stream().map(slot -> ((SlotReference) slot).getQualifiedNameWithBackquote()) .collect(Collectors.joining(", ")); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index c6c15fedaf5209..f036a73a6ed3e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -241,11 +241,12 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelation, List tableQualifier, CascadesContext cascadesContext) { - if (unboundRelation.getIndexInSqlString() != null) { + // for create view stmt replace tablename to ctl.db.tablename + unboundRelation.getIndexInSqlString().ifPresent(pair -> { StatementContext statementContext = cascadesContext.getStatementContext(); - statementContext.addIndexInSqlToString(unboundRelation.getIndexInSqlString(), - Utils.qualifiedNameWithBacktick(tableQualifier)); - } + statementContext.addIndexInSqlToString(pair, + Utils.qualifiedNameWithBackquote(tableQualifier)); + }); List qualifierWithoutTableName = Lists.newArrayList(); qualifierWithoutTableName.addAll(tableQualifier.subList(0, tableQualifier.size() - 1)); switch (table.getType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java index c63855157d28d0..3d11a7d011da15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java @@ -298,7 +298,7 @@ private static Supplier> buildInternalName( } } - public String getQualifiedNameWithBacktick() throws UnboundException { - return Utils.qualifiedNameWithBacktick(getQualifier(), getName()); + public String getQualifiedNameWithBackquote() throws UnboundException { + return Utils.qualifiedNameWithBackquote(getQualifier(), getName()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index 1d0bee75fb8178..ac8fcd6285a0bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -79,7 +79,6 @@ public class CreateViewInfo { private final String querySql; private final List simpleColumnDefinitions; private final List finalCols = Lists.newArrayList(); - private final List outputs = Lists.newArrayList(); private Plan analyzedPlan; /** constructor*/ @@ -102,8 +101,8 @@ public void init(ConnectContext ctx) throws UserException { OutermostPlanFinder outermostPlanFinder = new OutermostPlanFinder(); AtomicReference outermostPlan = new AtomicReference<>(); analyzedPlan.accept(outermostPlanFinder, outermostPlan); - outputs.addAll(outermostPlan.get().getOutput()); - createFinalCols(); + List outputs = outermostPlan.get().getOutput(); + createFinalCols(outputs); } /**validate*/ @@ -131,7 +130,7 @@ public void validate(ConnectContext ctx) throws UserException { public CreateViewStmt translateToLegacyStmt(ConnectContext ctx) { List cols = Lists.newArrayList(); for (SimpleColumnDefinition def : simpleColumnDefinitions) { - cols.add(def.transferToColWithComment()); + cols.add(def.translateToColWithComment()); } CreateViewStmt createViewStmt = new CreateViewStmt(ifNotExists, viewName.transferToTableName(), cols, comment, null); @@ -199,7 +198,7 @@ private String rewriteProjectsToUserDefineAlias(String resSql) { finder.getIndex().second + 1); } - private void createFinalCols() throws org.apache.doris.common.AnalysisException { + private void createFinalCols(List outputs) throws org.apache.doris.common.AnalysisException { if (simpleColumnDefinitions.isEmpty()) { for (Slot output : outputs) { Column column = new Column(output.getName(), output.getDataType().toCatalogDataType()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SimpleColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SimpleColumnDefinition.java index df6b88f6306701..e4e7f2013cb341 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SimpleColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/SimpleColumnDefinition.java @@ -42,7 +42,7 @@ public String getComment() { return comment; } - public ColWithComment transferToColWithComment() { + public ColWithComment translateToColWithComment() { return new ColWithComment(name, comment); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java index 5888fc9662f9bf..852e148ef1d9cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java @@ -129,18 +129,20 @@ public static String qualifiedName(List qualifier, String name) { } /** get qualified name with Backtick */ - public static String qualifiedNameWithBacktick(List qualifiers, String name) { + public static String qualifiedNameWithBackquote(List qualifiers, String name) { List fullName = new ArrayList<>(qualifiers); fullName.add(name); - return qualifiedNameWithBacktick(fullName); + return qualifiedNameWithBackquote(fullName); } - public static String qualifiedNameWithBacktick(List qualifiers) { - List qualifierWithBacktick = Lists.newArrayListWithCapacity(qualifiers.size()); + /** get qualified name with Backtick */ + public static String qualifiedNameWithBackquote(List qualifiers) { + List qualifierWithBackquote = Lists.newArrayListWithCapacity(qualifiers.size()); for (String qualifier : qualifiers) { - qualifierWithBacktick.add('`' + qualifier + '`'); + String escapeQualifier = qualifier.replace("`", "``"); + qualifierWithBackquote.add('`' + escapeQualifier + '`'); } - return StringUtils.join(qualifierWithBacktick, "."); + return StringUtils.join(qualifierWithBackquote, "."); } /** diff --git a/regression-test/data/ddl_p0/test_create_view_nereids.out b/regression-test/data/ddl_p0/test_create_view_nereids.out index 52f5bd5747d0ce..1fda4f63f57f79 100644 --- a/regression-test/data/ddl_p0/test_create_view_nereids.out +++ b/regression-test/data/ddl_p0/test_create_view_nereids.out @@ -182,3 +182,47 @@ test_view_star_except CREATE VIEW `test_view_star_except` COMMENT 'VIEW' AS sele -- !test_create_view_from_view_sql -- test_view_from_view CREATE VIEW `test_view_from_view` COMMENT 'VIEW' AS select `internal`.`regression_test_ddl_p0`.`test_view_with_as`.`pk` AS `c1`, `internal`.`regression_test_ddl_p0`.`test_view_with_as`.`a` AS `c2`, `internal`.`regression_test_ddl_p0`.`test_view_with_as`.`b` AS `c3` from `internal`.`regression_test_ddl_p0`.`test_view_with_as`; utf8mb4 utf8mb4_0900_bin +-- !test_backquote_in_view_define -- +\N 6 +1 2 +1 3 +1 4 +1 4 +1 7 +2 8 +3 5 +3 6 +3 9 +4 2 +5 \N +5 6 +5 6 +5 6 +5 8 +7 1 + +-- !test_backquote_in_view_define_sql -- +test_backquote_in_view_define CREATE VIEW `test_backquote_in_view_define` COMMENT 'VIEW' AS select a AS `ab``c`, b AS `c2` from `internal`.`regression_test_ddl_p0`.`mal_test_view`; utf8mb4 utf8mb4_0900_bin + +-- !test_backquote_in_table_alias -- +\N 6 +1 2 +1 3 +1 4 +1 4 +1 7 +2 8 +3 5 +3 6 +3 9 +4 2 +5 \N +5 6 +5 6 +5 6 +5 8 +7 1 + +-- !test_backquote_in_table_alias_sql -- +test_backquote_in_table_alias CREATE VIEW `test_backquote_in_table_alias` COMMENT 'VIEW' AS select `ab``c`.`a` AS `c1`, `ab``c`.`b` AS `c2` from (select a,b from `internal`.`regression_test_ddl_p0`.`mal_test_view`) `ab``c`; utf8mb4 utf8mb4_0900_bin + diff --git a/regression-test/suites/ddl_p0/test_create_view_nereids.groovy b/regression-test/suites/ddl_p0/test_create_view_nereids.groovy index 00c56076650d4b..48885a47fd157a 100644 --- a/regression-test/suites/ddl_p0/test_create_view_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_view_nereids.groovy @@ -266,4 +266,15 @@ suite("test_create_view_nereids") { qt_test_create_view_from_view "select * from test_view_from_view order by c1,c2,c3" qt_test_create_view_from_view_sql "show create view test_view_from_view" + // test backquote in name + + sql "drop view if exists test_backquote_in_view_define;" + sql "create view test_backquote_in_view_define(`ab``c`, c2) as select a,b from mal_test_view;" + qt_test_backquote_in_view_define "select * from test_backquote_in_view_define order by `ab``c`, c2;" + qt_test_backquote_in_view_define_sql "show create view test_backquote_in_view_define;" + + sql "drop view if exists test_backquote_in_table_alias;" + sql "create view test_backquote_in_table_alias(c1, c2) as select * from (select a,b from mal_test_view) `ab``c`;" + qt_test_backquote_in_table_alias "select * from test_backquote_in_table_alias order by c1, c2;" + qt_test_backquote_in_table_alias_sql "show create view test_backquote_in_table_alias;" } From ab8ade15b3ddcf441f651f60e22eec4c963510d0 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 8 Apr 2024 12:35:42 +0800 Subject: [PATCH 22/23] [feat](nereids) support create view in nereids --- .../apache/doris/nereids/parser/LogicalPlanBuilder.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c085403ba7b62c..31935c4a12123c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1400,8 +1400,8 @@ public Expression visitStar(StarContext ctx) { target = ImmutableList.of(); } return forCreateView - ? new UnboundStar(target, Optional.of(Pair.of(ctx.start.getStartIndex(), ctx.stop.getStopIndex()))) : - new UnboundStar(target); + ? new UnboundStar(target, Optional.of(Pair.of(ctx.start.getStartIndex(), ctx.stop.getStopIndex()))) + : new UnboundStar(target); }); } @@ -2891,8 +2891,9 @@ private LogicalPlan withSelectQuerySpecification( throw new ParseException("only column name is supported in except clause", selectColumnCtx); } UnboundStar star = forCreateView ? new UnboundStar(ImmutableList.of(), - Optional.of(Pair.of(selectColumnCtx.start.getStartIndex(), selectColumnCtx.stop.getStopIndex()))) : - new UnboundStar(ImmutableList.of()); + Optional.of(Pair.of(selectColumnCtx.start.getStartIndex(), + selectColumnCtx.stop.getStopIndex()))) + : new UnboundStar(ImmutableList.of()); project = new LogicalProject<>(ImmutableList.of(star), expressions, isDistinct, aggregate); } else { List projects = getNamedExpressions(selectColumnCtx.namedExpressionSeq()); From bf7dee4d25dd8197886c2c1f81b73605e341d8a9 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 8 Apr 2024 14:44:54 +0800 Subject: [PATCH 23/23] [feat](nereids) support create view in nereids --- .../plans/commands/info/CreateViewInfo.java | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index ac8fcd6285a0bc..632e884685652f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -66,7 +66,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; /** * CreateViewInfo @@ -98,10 +97,9 @@ public CreateViewInfo(boolean ifNotExists, TableNameInfo viewName, String commen /** init */ public void init(ConnectContext ctx) throws UserException { analyzeAndFillRewriteSqlMap(querySql, ctx); - OutermostPlanFinder outermostPlanFinder = new OutermostPlanFinder(); - AtomicReference outermostPlan = new AtomicReference<>(); - analyzedPlan.accept(outermostPlanFinder, outermostPlan); - List outputs = outermostPlan.get().getOutput(); + OutermostPlanFinderContext outermostPlanFinderContext = new OutermostPlanFinderContext(); + analyzedPlan.accept(OutermostPlanFinder.INSTANCE, outermostPlanFinderContext); + List outputs = outermostPlanFinderContext.outermostPlan.getOutput(); createFinalCols(outputs); } @@ -217,31 +215,36 @@ private void createFinalCols(List outputs) throws org.apache.doris.common. } } - private static class OutermostPlanFinder extends DefaultPlanVisitor> { - boolean found = false; + private static class OutermostPlanFinderContext { + public Plan outermostPlan = null; + public boolean found = false; + } + + private static class OutermostPlanFinder extends DefaultPlanVisitor { + public static final OutermostPlanFinder INSTANCE = new OutermostPlanFinder(); @Override - public Void visit(Plan plan, AtomicReference target) { - if (found) { + public Void visit(Plan plan, OutermostPlanFinderContext ctx) { + if (ctx.found) { return null; } - target.set(plan); - found = true; + ctx.outermostPlan = plan; + ctx.found = true; return null; } @Override public Void visitLogicalCTEAnchor(LogicalCTEAnchor cteAnchor, - AtomicReference target) { - if (found) { + OutermostPlanFinderContext ctx) { + if (ctx.found) { return null; } - return super.visit(cteAnchor, target); + return super.visit(cteAnchor, ctx); } @Override public Void visitLogicalCTEProducer(LogicalCTEProducer cteProducer, - AtomicReference target) { + OutermostPlanFinderContext ctx) { return null; } }