diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 67f3e7997fa9ef..acf3e4ec566b4f 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1450,7 +1450,9 @@ rowConstructor ; rowConstructorItem - : namedExpression | DEFAULT + : constant // duplicate constant rule for improve the parse of `insert into tbl values` + | DEFAULT + | namedExpression ; predicate @@ -1644,7 +1646,7 @@ constant | LEFT_BRACE (items+=constant COLON items+=constant)? (COMMA items+=constant COLON items+=constant)* RIGHT_BRACE #mapLiteral | LEFT_BRACE items+=constant (COMMA items+=constant)* RIGHT_BRACE #structLiteral - | PLACEHOLDER #placeholder + | PLACEHOLDER #placeholder ; comparisonOperator diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 1486f03e269b13..5944ce693da2f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -473,9 +473,16 @@ public void setCurrentRootRewriteJobContext(RootRewriteJobContext currentRootRew this.currentRootRewriteJobContext = Optional.ofNullable(currentRootRewriteJobContext); } + /** showPlanProcess */ public boolean showPlanProcess() { Boolean show = showPlanProcess.get(); - return show != null && show; + if (show != null && show) { + return true; + } + if (parent.isPresent()) { + return parent.get().showPlanProcess(); + } + return false; } /** set showPlanProcess in task scope */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 85e40c6c7b37fb..bd2b67efaecf62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -98,17 +98,19 @@ */ public class NereidsPlanner extends Planner { public static final Logger LOG = LogManager.getLogger(NereidsPlanner.class); + + protected Plan parsedPlan; + protected Plan analyzedPlan; + protected Plan rewrittenPlan; + protected Plan optimizedPlan; + protected PhysicalPlan physicalPlan; + private CascadesContext cascadesContext; private final StatementContext statementContext; private final List scanNodeList = Lists.newArrayList(); private final List physicalRelations = Lists.newArrayList(); private DescriptorTable descTable; - private Plan parsedPlan; - private Plan analyzedPlan; - private Plan rewrittenPlan; - private Plan optimizedPlan; - private PhysicalPlan physicalPlan; private FragmentIdMapping distributedPlans; // The cost of optimized plan private double cost = 0; @@ -358,7 +360,7 @@ protected void collectAndLockTable(boolean showPlanProcess) { } } - private void analyze(boolean showPlanProcess) { + protected void analyze(boolean showPlanProcess) { if (LOG.isDebugEnabled()) { LOG.debug("Start analyze plan"); } @@ -378,7 +380,7 @@ private void analyze(boolean showPlanProcess) { /** * Logical plan rewrite based on a series of heuristic rules. */ - private void rewrite(boolean showPlanProcess) { + protected void rewrite(boolean showPlanProcess) { if (LOG.isDebugEnabled()) { LOG.debug("Start rewrite plan"); } @@ -536,7 +538,7 @@ private void distribute(PhysicalPlan physicalPlan, ExplainLevel explainLevel) { } } - private PhysicalPlan postProcess(PhysicalPlan physicalPlan) { + protected PhysicalPlan postProcess(PhysicalPlan physicalPlan) { return new PlanPostProcessors(cascadesContext).process(physicalPlan); } @@ -553,7 +555,7 @@ public Group getRoot() { return cascadesContext.getMemo().getRoot(); } - private PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) { + protected PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) { if (nthPlan <= 1) { cost = rootGroup.getLowestCostPlan(physicalProperties).orElseThrow( () -> new AnalysisException("lowestCostPlans with physicalProperties(" @@ -606,6 +608,9 @@ private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physical } private long getGarbageCollectionTime() { + if (!ConnectContext.get().getSessionVariable().enableProfile()) { + return 0; + } List gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans(); long initialGCTime = 0; for (GarbageCollectorMXBean gcBean : gcMxBeans) { @@ -886,7 +891,7 @@ private boolean showPlanProcess(ExplainOptions explainOptions) { return explainOptions != null && explainOptions.showPlanProcess(); } - private void keepOrShowPlanProcess(boolean showPlanProcess, Runnable task) { + protected void keepOrShowPlanProcess(boolean showPlanProcess, Runnable task) { if (showPlanProcess) { cascadesContext.withPlanProcess(showPlanProcess, task); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java new file mode 100644 index 00000000000000..42d637d676fae2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundInlineTable.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; +import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** UnboundInlineTable */ +public class UnboundInlineTable extends LogicalLeaf implements InlineTable, BlockFuncDepsPropagation, UnboundPlan { + private final List> constantExprsList; + + public UnboundInlineTable(List> constantExprsList) { + super(PlanType.LOGICAL_UNBOUND_INLINE_TABLE, Optional.empty(), Optional.empty()); + this.constantExprsList = Utils.fastToImmutableList( + Objects.requireNonNull(constantExprsList, "constantExprsList can not be null") + ); + } + + public List> getConstantExprsList() { + return constantExprsList; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitUnboundInlineTable(this, context); + } + + @Override + public List getExpressions() { + ImmutableList.Builder expressions = ImmutableList.builderWithExpectedSize( + constantExprsList.size() * constantExprsList.get(0).size()); + + for (List namedExpressions : constantExprsList) { + expressions.addAll(namedExpressions); + } + + return expressions.build(); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return this; + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return this; + } + + @Override + public List computeOutput() { + throw new UnboundException("output"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPlan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPlan.java new file mode 100644 index 00000000000000..2b743f958aaa02 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundPlan.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.UnboundLogicalProperties; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; + +/** UnboundPlan */ +public interface UnboundPlan extends Plan { + @Override + default LogicalProperties computeLogicalProperties() { + return UnboundLogicalProperties.INSTANCE; + } + + @Override + default List computeOutput() { + throw new UnboundException("output"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java index 0e528227dc9742..8cf32648d55f05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java @@ -34,6 +34,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import org.apache.commons.lang3.StringUtils; import java.util.List; import java.util.Objects; @@ -176,4 +177,12 @@ public LogicalProperties computeLogicalProperties() { public List computeOutput() { throw new UnboundException("output"); } + + @Override + public String toString() { + return Utils.toSqlString("UnboundTableSink", + "nameParts", StringUtils.join(nameParts, "."), + "colNames", colNames, + "hints", hints); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 82d3f36ea6b86b..8ea9de280d6f4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -208,6 +208,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -2114,17 +2115,22 @@ public PlanFragment visitPhysicalSetOperation( } setOperationNode.setNereidsId(setOperation.getId()); - setOperation.getRegularChildrenOutputs().stream() - .map(o -> o.stream() - .map(e -> ExpressionTranslator.translate(e, context)) - .collect(ImmutableList.toImmutableList())) - .forEach(setOperationNode::addResultExprLists); + for (List regularChildrenOutput : setOperation.getRegularChildrenOutputs()) { + Builder translateOutputs = ImmutableList.builderWithExpectedSize(regularChildrenOutput.size()); + for (SlotReference childOutput : regularChildrenOutput) { + translateOutputs.add(ExpressionTranslator.translate(childOutput, context)); + } + setOperationNode.addResultExprLists(translateOutputs.build()); + } + if (setOperation instanceof PhysicalUnion) { - ((PhysicalUnion) setOperation).getConstantExprsList().stream() - .map(l -> l.stream() - .map(e -> ExpressionTranslator.translate(e, context)) - .collect(ImmutableList.toImmutableList())) - .forEach(setOperationNode::addConstExprList); + for (List unionConsts : ((PhysicalUnion) setOperation).getConstantExprsList()) { + Builder translateConsts = ImmutableList.builderWithExpectedSize(unionConsts.size()); + for (NamedExpression unionConst : unionConsts) { + translateConsts.add(ExpressionTranslator.translate(unionConst, context)); + } + setOperationNode.addConstExprList(translateConsts.build()); + } } for (PlanFragment childFragment : childrenFragments) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 6d21441c95f3a4..0465f44ef13434 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -213,7 +213,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundFunction; -import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundInlineTable; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundSlot; @@ -367,6 +367,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand; import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand; @@ -454,10 +455,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; -import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalPreAggOnHint; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -617,7 +618,7 @@ public LogicalPlan visitInsertTable(InsertTableContext ctx) { } else { throw new ParseException("tableName and tableId cannot both be null"); } - Optional labelName = ctx.labelName == null ? Optional.empty() : Optional.of(ctx.labelName.getText()); + Optional labelName = (ctx.labelName == null) ? Optional.empty() : Optional.of(ctx.labelName.getText()); List colNames = ctx.cols == null ? ImmutableList.of() : visitIdentifierList(ctx.cols); // TODO visit partitionSpecCtx LogicalPlan plan = visitQuery(ctx.query()); @@ -648,7 +649,7 @@ public LogicalPlan visitInsertTable(InsertTableContext ctx) { command = new InsertOverwriteTableCommand(sink, labelName, cte); } else { if (ConnectContext.get() != null && ConnectContext.get().isTxnModel() - && sink.child() instanceof LogicalInlineTable + && sink.child() instanceof InlineTable && sink.child().getExpressions().stream().allMatch(Expression::isConstant)) { // FIXME: In legacy, the `insert into select 1` is handled as `insert into values`. // In nereids, the original way is throw an AnalysisException and fallback to legacy. @@ -1442,8 +1443,8 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte if (columnCtx.EXCEPT() != null) { throw new ParseException("select-except cannot be used in one row relation", selectCtx); } - relation = new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), - ImmutableList.of(new UnboundAlias(Literal.of(0)))); + relation = new LogicalOneRowRelation(StatementScopeIdGenerator.newRelationId(), + ImmutableList.of(new Alias(Literal.of(0)))); } else { relation = visitFromClause(ctx.fromClause()); } @@ -1475,10 +1476,13 @@ public LogicalPlan visitRegularQuerySpecification(RegularQuerySpecificationConte @Override public LogicalPlan visitInlineTable(InlineTableContext ctx) { - List> values = ctx.rowConstructor().stream() - .map(this::visitRowConstructor) - .collect(ImmutableList.toImmutableList()); - return new LogicalInlineTable(values); + List rowConstructorContexts = ctx.rowConstructor(); + ImmutableList.Builder> rows + = ImmutableList.builderWithExpectedSize(rowConstructorContexts.size()); + for (RowConstructorContext rowConstructorContext : rowConstructorContexts) { + rows.add(visitRowConstructor(rowConstructorContext)); + } + return new UnboundInlineTable(rows.build()); } /** @@ -1623,6 +1627,8 @@ public NamedExpression visitNamedExpression(NamedExpressionContext ctx) { if (ctx.identifierOrText() == null) { if (expression instanceof NamedExpression) { return (NamedExpression) expression; + } else if (expression instanceof Literal) { + return new Alias(expression); } else { int start = ctx.expression().start.getStartIndex(); int stop = ctx.expression().stop.getStopIndex(); @@ -1636,6 +1642,9 @@ public NamedExpression visitNamedExpression(NamedExpressionContext ctx) { } } String alias = visitIdentifierOrText(ctx.identifierOrText()); + if (expression instanceof Literal) { + return new Alias(expression, alias); + } return new UnboundAlias(expression, alias); }); } @@ -2575,14 +2584,21 @@ public Expression visitParenthesizedExpression(ParenthesizedExpressionContext ct @Override public List visitRowConstructor(RowConstructorContext ctx) { - return ctx.rowConstructorItem().stream() - .map(this::visitRowConstructorItem) - .collect(ImmutableList.toImmutableList()); + List rowConstructorItemContexts = ctx.rowConstructorItem(); + ImmutableList.Builder columns + = ImmutableList.builderWithExpectedSize(rowConstructorItemContexts.size()); + for (RowConstructorItemContext rowConstructorItemContext : rowConstructorItemContexts) { + columns.add(visitRowConstructorItem(rowConstructorItemContext)); + } + return columns.build(); } @Override public NamedExpression visitRowConstructorItem(RowConstructorItemContext ctx) { - if (ctx.DEFAULT() != null) { + ConstantContext constant = ctx.constant(); + if (constant != null) { + return new Alias((Expression) constant.accept(this)); + } else if (ctx.DEFAULT() != null) { return new DefaultValueSlot(); } else { return visitNamedExpression(ctx.namedExpression()); @@ -3138,14 +3154,6 @@ private LogicalPlan withLimit(LogicalPlan input, Optional li }); } - private UnboundOneRowRelation withOneRowRelation(SelectColumnClauseContext selectCtx) { - return ParserUtils.withOrigin(selectCtx, () -> { - // fromClause does not exists. - List projects = getNamedExpressions(selectCtx.namedExpressionSeq()); - return new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), projects); - }); - } - /** * Add a regular (SELECT) query specification to a logical plan. The query specification * is the core of the logical plan, this is where sourcing (FROM clause), projection (SELECT), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 4ed71bbbc14673..c273f50b04ac44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -56,6 +56,7 @@ import java.lang.reflect.Method; import java.util.BitSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -326,37 +327,40 @@ private T parse(String sql, Function parseFu private T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder, Function parseFunction) { - ParserRuleContext tree = toAst(sql, parseFunction); + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, parseFunction); LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null - ? new LogicalPlanBuilder(getHintMap(sql, DorisParser::selectHint)) : logicalPlanBuilder; + ? new LogicalPlanBuilder(getHintMap(sql, tokenStream, DorisParser::selectHint)) + : logicalPlanBuilder; return (T) realLogicalPlanBuilder.visit(tree); } public LogicalPlan parseForCreateView(String sql) { - ParserRuleContext tree = toAst(sql, DorisParser::singleStatement); + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement); LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView( - getHintMap(sql, DorisParser::selectHint)); + getHintMap(sql, tokenStream, DorisParser::selectHint)); return (LogicalPlan) realLogicalPlanBuilder.visit(tree); } + /** parseForSyncMv */ public Optional parseForSyncMv(String sql) { - ParserRuleContext tree = toAst(sql, DorisParser::singleStatement); + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement); LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv( - getHintMap(sql, DorisParser::selectHint)); + getHintMap(sql, tokenStream, DorisParser::selectHint)); logicalPlanBuilderForSyncMv.visit(tree); return logicalPlanBuilderForSyncMv.getQuerySql(); } /** get hint map */ - public static Map getHintMap(String sql, + public static Map getHintMap(String sql, CommonTokenStream hintTokenStream, Function parseFunction) { // parse hint first round - DorisLexer hintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); - CommonTokenStream hintTokenStream = new CommonTokenStream(hintLexer); - Map selectHintMap = Maps.newHashMap(); - Token hintToken = hintTokenStream.getTokenSource().nextToken(); + Iterator tokenIterator = hintTokenStream.getTokens().iterator(); + Token hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null; while (hintToken != null && hintToken.getType() != DorisLexer.EOF) { if (hintToken.getChannel() == 2 && sql.charAt(hintToken.getStartIndex() + 2) == '+') { String hintSql = sql.substring(hintToken.getStartIndex() + 3, hintToken.getStopIndex() + 1); @@ -366,15 +370,19 @@ public static Map getHintMap(String sql, ParserRuleContext hintContext = parseFunction.apply(hintParser); selectHintMap.put(hintToken.getStartIndex(), hintContext); } - hintToken = hintTokenStream.getTokenSource().nextToken(); + hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null; } return selectHintMap; } + public static ParserRuleContext toAst( + String sql, Function parseFunction) { + return toAst(parseAllTokens(sql), parseFunction); + } + /** toAst */ - public static ParserRuleContext toAst(String sql, Function parseFunction) { - DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); - CommonTokenStream tokenStream = new CommonTokenStream(lexer); + public static ParserRuleContext toAst( + CommonTokenStream tokenStream, Function parseFunction) { DorisParser parser = new DorisParser(tokenStream); parser.addParseListener(POST_PROCESSOR); @@ -405,9 +413,7 @@ public static ParserRuleContext toAst(String sql, Function> aggregate() { default PatternDescriptor> aggregate(PatternDescriptor child) { return new PatternDescriptor(new TypePattern(Aggregate.class, child.pattern), defaultPromise()); } + + /** + * create a aggregate pattern. + */ + default PatternDescriptor inlineTable() { + return new PatternDescriptor(new TypePattern(InlineTable.class), defaultPromise()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index b7cbe5343d1129..387b4999a3436a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -60,6 +60,7 @@ public enum RuleType { BINDING_UNBOUND_TVF_RELATION_FUNCTION(RuleTypeClass.REWRITE), BINDING_SET_OPERATION_SLOT(RuleTypeClass.REWRITE), BINDING_INLINE_TABLE_SLOT(RuleTypeClass.REWRITE), + LOGICAL_INLINE_TABLE_TO_LOGICAL_UNION_OR_ONE_ROW_RELATION(RuleTypeClass.REWRITE), COUNT_LITERAL_REWRITE(RuleTypeClass.REWRITE), SUM_LITERAL_REWRITE(RuleTypeClass.REWRITE), @@ -486,8 +487,7 @@ public enum RuleType { IMPLEMENTATION_SENTINEL(RuleTypeClass.IMPLEMENTATION), // sentinel, use to count rules - SENTINEL(RuleTypeClass.SENTINEL), - ; + SENTINEL(RuleTypeClass.SENTINEL); private final RuleTypeClass ruleTypeClass; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 3bf5b360b4746b..9e77ec863dcd08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -66,6 +66,7 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.algebra.SetOperation; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -74,7 +75,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; -import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; @@ -182,7 +182,7 @@ protected boolean condition(Rule rule, Plan plan) { logicalHaving(any().whenNot(Aggregate.class::isInstance)).thenApply(this::bindHaving) ), RuleType.BINDING_INLINE_TABLE_SLOT.build( - logicalInlineTable().thenApply(this::bindInlineTable) + inlineTable().thenApply(this::bindInlineTable) ), RuleType.BINDING_ONE_ROW_RELATION_SLOT.build( // we should bind UnboundAlias in the UnboundOneRowRelation @@ -336,24 +336,24 @@ private LogicalOneRowRelation bindOneRowRelation(MatchingContext ctx) { - LogicalInlineTable logicalInlineTable = ctx.root; + private LogicalPlan bindInlineTable(MatchingContext ctx) { + InlineTable inlineTable = ctx.root; // ensure all expressions are valid. + List> constantExprsList = inlineTable.getConstantExprsList(); List relations - = Lists.newArrayListWithCapacity(logicalInlineTable.getConstantExprsList().size()); - for (int i = 0; i < logicalInlineTable.getConstantExprsList().size(); i++) { - for (NamedExpression constantExpr : logicalInlineTable.getConstantExprsList().get(i)) { + = Lists.newArrayListWithCapacity(constantExprsList.size()); + for (int i = 0; i < constantExprsList.size(); i++) { + List row = constantExprsList.get(i); + for (NamedExpression constantExpr : row) { if (constantExpr instanceof DefaultValueSlot) { throw new AnalysisException("Default expression" + " can't exist in SELECT statement at row " + (i + 1)); } } - relations.add(new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), - logicalInlineTable.getConstantExprsList().get(i))); + relations.add(new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), row)); } // construct union all tree - return LogicalPlanBuilder.reduceToLogicalPlanTree(0, relations.size() - 1, - relations, Qualifier.ALL); + return LogicalPlanBuilder.reduceToLogicalPlanTree(0, relations.size() - 1, relations, Qualifier.ALL); } private LogicalHaving bindHaving(MatchingContext> ctx) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 5bf20d7069586b..e975b4914cfe89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -75,6 +75,7 @@ import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; @@ -227,7 +228,7 @@ private Plan bindOlapTableSink(MatchingContext> ctx) { private LogicalProject getOutputProjectByCoercion(List tableSchema, LogicalPlan child, Map columnToOutput) { - List fullOutputExprs = ImmutableList.copyOf(columnToOutput.values()); + List fullOutputExprs = Utils.fastToImmutableList(columnToOutput.values()); if (child instanceof LogicalOneRowRelation) { // remove default value slot in one row relation child = ((LogicalOneRowRelation) child).withProjects(((LogicalOneRowRelation) child) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index e69f93bf84551e..5aa43fb05b60e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -43,6 +43,7 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableSet; import java.util.Collection; @@ -82,7 +83,8 @@ public List buildRules() { new HavingExpressionRewrite().build()); } - private class GenerateExpressionRewrite extends OneRewriteRuleFactory { + /** GenerateExpressionRewrite */ + public class GenerateExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalGenerate().thenApply(ctx -> { @@ -100,7 +102,8 @@ public Rule build() { } } - private class OneRowRelationExpressionRewrite extends OneRewriteRuleFactory { + /** OneRowRelationExpressionRewrite */ + public class OneRowRelationExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalOneRowRelation().thenApply(ctx -> { @@ -108,19 +111,25 @@ public Rule build() { List projects = oneRowRelation.getProjects(); ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); - List newProjects = projects - .stream() - .map(expr -> (NamedExpression) rewriter.rewrite(expr, context)) - .collect(ImmutableList.toImmutableList()); - if (projects.equals(newProjects)) { - return oneRowRelation; + Builder rewrittenExprs + = ImmutableList.builderWithExpectedSize(projects.size()); + boolean changed = false; + for (NamedExpression project : projects) { + NamedExpression newProject = (NamedExpression) rewriter.rewrite(project, context); + if (!changed && !project.deepEquals(newProject)) { + changed = true; + } + rewrittenExprs.add(newProject); } - return new LogicalOneRowRelation(oneRowRelation.getRelationId(), newProjects); + return changed + ? new LogicalOneRowRelation(oneRowRelation.getRelationId(), rewrittenExprs.build()) + : oneRowRelation; }).toRule(RuleType.REWRITE_ONE_ROW_RELATION_EXPRESSION); } } - private class ProjectExpressionRewrite extends OneRewriteRuleFactory { + /** ProjectExpressionRewrite */ + public class ProjectExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalProject().thenApply(ctx -> { @@ -136,7 +145,8 @@ public Rule build() { } } - private class FilterExpressionRewrite extends OneRewriteRuleFactory { + /** FilterExpressionRewrite */ + public class FilterExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalFilter().thenApply(ctx -> { @@ -152,7 +162,8 @@ public Rule build() { } } - private class AggExpressionRewrite extends OneRewriteRuleFactory { + /** AggExpressionRewrite */ + public class AggExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalAggregate().thenApply(ctx -> { @@ -172,7 +183,8 @@ public Rule build() { } } - private class JoinExpressionRewrite extends OneRewriteRuleFactory { + /** JoinExpressionRewrite */ + public class JoinExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalJoin().thenApply(ctx -> { @@ -227,7 +239,8 @@ private Pair> rewriteConjuncts(List conjun } } - private class SortExpressionRewrite extends OneRewriteRuleFactory { + /** SortExpressionRewrite */ + public class SortExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { @@ -248,7 +261,8 @@ public Rule build() { } } - private class HavingExpressionRewrite extends OneRewriteRuleFactory { + /** HavingExpressionRewrite */ + public class HavingExpressionRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalHaving().thenApply(ctx -> { @@ -264,7 +278,8 @@ public Rule build() { } } - private class LogicalRepeatRewrite extends OneRewriteRuleFactory { + /** LogicalRepeatRewrite */ + public class LogicalRepeatRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalRepeat().thenApply(ctx -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java index 6aa4529ddd4ab6..6d5a70139ab19c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ConvertAggStateCast.java @@ -47,7 +47,8 @@ public List> buildRules() { ); } - private static Expression convert(Cast cast) { + /** convert */ + public static Expression convert(Cast cast) { Expression child = cast.child(); DataType originalType = child.getDataType(); DataType targetType = cast.getDataType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java index 424673870b894d..38cc807274ff1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java @@ -24,7 +24,9 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.analyzer.UnboundVariable; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; import org.apache.doris.nereids.rules.expression.AbstractExpressionRewriteRule; import org.apache.doris.nereids.rules.expression.ExpressionListenerMatcher; import org.apache.doris.nereids.rules.expression.ExpressionMatchingContext; @@ -53,6 +55,7 @@ import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; +import org.apache.doris.nereids.trees.expressions.Variable; import org.apache.doris.nereids.trees.expressions.WhenClause; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; @@ -222,6 +225,12 @@ public Expression visitMatch(Match match, ExpressionRewriteContext context) { return super.visitMatch(match, context); } + @Override + public Expression visitUnboundVariable(UnboundVariable unboundVariable, ExpressionRewriteContext context) { + Variable variable = ExpressionAnalyzer.resolveUnboundVariable(unboundVariable); + return variable.getRealExpression(); + } + @Override public Expression visitEncryptKeyRef(EncryptKeyRef encryptKeyRef, ExpressionRewriteContext context) { String dbName = encryptKeyRef.getDbName(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java index 4144d2ab11c36e..204bd3b4d611be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java @@ -104,6 +104,7 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.time.DateTimeException; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; @@ -224,7 +225,7 @@ private ColumnStatistic castMinMax(ColumnStatistic colStats, DataType targetType long min = dateMinLiteral.getValue(); builder.setMinValue(min); builder.setMinExpr(dateMinLiteral.toLegacyLiteral()); - } catch (AnalysisException e) { + } catch (AnalysisException | DateTimeException e) { convertSuccess = false; } } @@ -235,7 +236,7 @@ private ColumnStatistic castMinMax(ColumnStatistic colStats, DataType targetType long max = dateMaxLiteral.getValue(); builder.setMaxValue(max); builder.setMaxExpr(dateMaxLiteral.toLegacyLiteral()); - } catch (AnalysisException e) { + } catch (AnalysisException | DateTimeException e) { convertSuccess = false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java index 56bbcb5ca962ec..a41fd5f7acc3ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateLiteral.java @@ -274,7 +274,7 @@ static Result normalize(String s) { /** parseDateLiteral */ public static Result parseDateLiteral(String s) { - Result parseResult = parseDateTime(s); + Result parseResult = parseDateTime(s); if (parseResult.isError()) { return parseResult.cast(); } @@ -290,14 +290,21 @@ public static Result parseDateLiteral(String s) } /** parseDateTime */ - public static Result parseDateTime(String s) { + public static Result parseDateTime(String s) { String originalString = s; try { // fast parse '2022-01-01' - if (s.length() == 10 && s.charAt(4) == '-' && s.charAt(7) == '-') { - TemporalAccessor date = fastParseDate(s); - if (date != null) { - return Result.ok(date); + if ((s.length() == 10 || s.length() == 19) && s.charAt(4) == '-' && s.charAt(7) == '-') { + if (s.length() == 10) { + TemporalAccessor date = fastParseDate(s); + if (date != null) { + return Result.ok(date); + } + } else if (s.charAt(10) == ' ' && s.charAt(13) == ':' && s.charAt(16) == ':') { + TemporalAccessor date = fastParseDateTime(s); + if (date != null) { + return Result.ok(date); + } } } @@ -570,6 +577,21 @@ private static TemporalAccessor fastParseDate(String date) { } } + private static TemporalAccessor fastParseDateTime(String date) { + Integer year = readNextInt(date, 0, 4); + Integer month = readNextInt(date, 5, 2); + Integer day = readNextInt(date, 8, 2); + Integer hour = readNextInt(date, 11, 2); + Integer minute = readNextInt(date, 14, 2); + Integer second = readNextInt(date, 17, 2); + + if (year != null && month != null && day != null && hour != null && minute != null && second != null) { + return LocalDateTime.of(year, month, day, hour, minute, second); + } else { + return null; + } + } + private static Integer readNextInt(String str, int offset, int readLength) { int value = 0; int realReadLength = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index 17c5678b05170b..d62a027fb08f2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -133,7 +133,7 @@ public static int determineScale(String s) { /** parseDateTimeLiteral */ public static Result parseDateTimeLiteral(String s, boolean isV2) { - Result parseResult = parseDateTime(s); + Result parseResult = parseDateTime(s); if (parseResult.isError()) { return parseResult.cast(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Explainable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Explainable.java index 46771392e59cd9..77eef860b98a77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Explainable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Explainable.java @@ -17,11 +17,20 @@ package org.apache.doris.nereids.trees.plans; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.qe.ConnectContext; +import java.util.Optional; + /** * plan can be explained. */ public interface Explainable { Plan getExplainPlan(ConnectContext ctx) throws Exception; + + default Optional getExplainPlanner(LogicalPlan logicalPlan, StatementContext ctx) throws Exception { + return Optional.empty(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 559d62c85fd58c..2fe0f372352c5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -43,6 +43,7 @@ public enum PlanType { LOGICAL_UNBOUND_ONE_ROW_RELATION, LOGICAL_UNBOUND_RELATION, LOGICAL_UNBOUND_TVF_RELATION, + LOGICAL_UNBOUND_INLINE_TABLE, // logical sinks LOGICAL_FILE_SINK, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/InlineTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/InlineTable.java new file mode 100644 index 00000000000000..0aded14ca77119 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/InlineTable.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.algebra; + +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.LeafPlan; + +import java.util.List; + +/** InlineTable */ +public interface InlineTable extends LeafPlan { + List> getConstantExprsList(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index e3f2f1d732ae5a..ea805f6cb0ceb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -79,12 +79,16 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { if (!(logicalPlan instanceof Explainable)) { throw new AnalysisException(logicalPlan.getClass().getSimpleName() + " cannot be explained"); } - explainPlan = ((LogicalPlan) ((Explainable) logicalPlan).getExplainPlan(ctx)); + Explainable explainable = (Explainable) logicalPlan; + explainPlan = ((LogicalPlan) explainable.getExplainPlan(ctx)); + NereidsPlanner planner = explainable.getExplainPlanner(explainPlan, ctx.getStatementContext()).orElseGet(() -> + new NereidsPlanner(ctx.getStatementContext()) + ); + LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(explainPlan, ctx.getStatementContext()); ExplainOptions explainOptions = new ExplainOptions(level, showPlanProcess); logicalPlanAdapter.setIsExplain(explainOptions); executor.setParsedStmt(logicalPlanAdapter); - NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); if (ctx.getSessionVariable().isEnableMaterializedViewRewrite()) { ctx.getStatementContext().addPlannerHook(InitMaterializationContextHook.INSTANCE); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java index 414f6ed07f7ea1..b31d03cadcdb61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java @@ -26,19 +26,22 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.NoForward; -import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; @@ -69,16 +72,34 @@ public class BatchInsertIntoTableCommand extends Command implements NoForward, E public static final Logger LOG = LogManager.getLogger(BatchInsertIntoTableCommand.class); - private LogicalPlan logicalQuery; + private LogicalPlan originLogicalQuery; + private Optional logicalQuery; public BatchInsertIntoTableCommand(LogicalPlan logicalQuery) { super(PlanType.BATCH_INSERT_INTO_TABLE_COMMAND); - this.logicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); + this.originLogicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); + this.logicalQuery = Optional.empty(); + } + + public LogicalPlan getLogicalQuery() { + return logicalQuery.orElse(originLogicalQuery); } @Override public Plan getExplainPlan(ConnectContext ctx) throws Exception { - return InsertUtils.getPlanForExplain(ctx, this.logicalQuery); + Optional analyzeContext = Optional.of( + CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY) + ); + return InsertUtils.getPlanForExplain(ctx, analyzeContext, getLogicalQuery()); + } + + @Override + public Optional getExplainPlanner(LogicalPlan logicalPlan, StatementContext ctx) throws Exception { + ConnectContext connectContext = ctx.getConnectContext(); + TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, connectContext); + boolean supportFastInsertIntoValues + = InsertUtils.supportFastInsertIntoValues(logicalPlan, targetTableIf, connectContext); + return Optional.of(new FastInsertIntoValuesPlanner(ctx, supportFastInsertIntoValues)); } @Override @@ -88,19 +109,32 @@ public R accept(PlanVisitor visitor, C context) { @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { - UnboundTableSink unboundTableSink = (UnboundTableSink) logicalQuery; + UnboundTableSink unboundTableSink = (UnboundTableSink) originLogicalQuery; Plan query = unboundTableSink.child(); - if (!(query instanceof LogicalInlineTable)) { + if (!(query instanceof InlineTable)) { throw new AnalysisException("Insert into ** select is not supported in a transaction"); } PhysicalOlapTableSink sink; - TableIf targetTableIf = InsertUtils.getTargetTable(logicalQuery, ctx); + TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, ctx); targetTableIf.readLock(); try { - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, Optional.empty()); - LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext()); - NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); + StatementContext statementContext = ctx.getStatementContext(); + Optional analyzeContext = Optional.of( + CascadesContext.initContext(statementContext, originLogicalQuery, PhysicalProperties.ANY) + ); + + this.logicalQuery = Optional.of((LogicalPlan) InsertUtils.normalizePlan( + originLogicalQuery, targetTableIf, analyzeContext, Optional.empty() + )); + + LogicalPlan logicalQuery = this.logicalQuery.get(); + LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, statementContext); + + boolean supportFastInsertIntoValues + = InsertUtils.supportFastInsertIntoValues(logicalQuery, targetTableIf, ctx); + FastInsertIntoValuesPlanner planner = new FastInsertIntoValuesPlanner( + statementContext, supportFastInsertIntoValues, true); planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); executor.checkBlockRules(); if (ctx.getConnectType() == ConnectType.MYSQL && ctx.getMysqlChannel() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/FastInsertIntoValuesPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/FastInsertIntoValuesPlanner.java new file mode 100644 index 00000000000000..18dcbf25d28aa6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/FastInsertIntoValuesPlanner.java @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.memo.Group; +import org.apache.doris.nereids.memo.GroupId; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.implementation.LogicalOlapTableSinkToPhysicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; +import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; + +import java.util.concurrent.atomic.AtomicReference; + +/** FastInsertIntoValuesPlanner */ +public class FastInsertIntoValuesPlanner extends NereidsPlanner { + private static final Rule toPhysicalOlapTableSink = new LogicalOlapTableSinkToPhysicalOlapTableSink() + .build(); + protected final boolean fastInsertIntoValues; + protected final boolean batchInsert; + private final AtomicReference rootGroupRef = new AtomicReference<>(); + + public FastInsertIntoValuesPlanner(StatementContext statementContext, boolean fastInsertIntoValues) { + this(statementContext, fastInsertIntoValues, false); + } + + public FastInsertIntoValuesPlanner( + StatementContext statementContext, boolean fastInsertIntoValues, boolean batchInsert) { + super(statementContext); + this.fastInsertIntoValues = fastInsertIntoValues; + this.batchInsert = batchInsert; + } + + @Override + protected void analyze(boolean showPlanProcess) { + if (!fastInsertIntoValues) { + super.analyze(showPlanProcess); + return; + } + CascadesContext cascadesContext = getCascadesContext(); + keepOrShowPlanProcess(showPlanProcess, () -> { + InsertIntoValuesAnalyzer analyzer = new InsertIntoValuesAnalyzer(cascadesContext, batchInsert); + analyzer.execute(); + }); + } + + @Override + protected void rewrite(boolean showPlanProcess) { + if (!fastInsertIntoValues) { + super.rewrite(showPlanProcess); + } + } + + @Override + protected void optimize() { + if (!fastInsertIntoValues) { + super.optimize(); + return; + } + + DefaultPlanRewriter optimizer = new DefaultPlanRewriter() { + @Override + public Plan visitLogicalUnion(LogicalUnion logicalUnion, Void context) { + logicalUnion = (LogicalUnion) super.visitLogicalUnion(logicalUnion, context); + + return new PhysicalUnion(logicalUnion.getQualifier(), + logicalUnion.getOutputs(), + logicalUnion.getRegularChildrenOutputs(), + logicalUnion.getConstantExprsList(), + logicalUnion.getLogicalProperties(), + logicalUnion.children() + ); + } + + @Override + public Plan visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation, Void context) { + return new PhysicalOneRowRelation( + oneRowRelation.getRelationId(), + oneRowRelation.getProjects(), + oneRowRelation.getLogicalProperties()); + } + + @Override + public Plan visitLogicalProject(LogicalProject logicalProject, Void context) { + logicalProject = + (LogicalProject) super.visitLogicalProject(logicalProject, context); + + return new PhysicalProject<>( + logicalProject.getProjects(), + logicalProject.getLogicalProperties(), + logicalProject.child() + ); + } + + @Override + public Plan visitLogicalOlapTableSink(LogicalOlapTableSink olapTableSink, + Void context) { + olapTableSink = + (LogicalOlapTableSink) super.visitLogicalOlapTableSink(olapTableSink, context); + return toPhysicalOlapTableSink + .transform(olapTableSink, getCascadesContext()) + .get(0); + } + }; + + PhysicalPlan physicalPlan = + (PhysicalPlan) getCascadesContext().getRewritePlan().accept(optimizer, null); + + super.physicalPlan = physicalPlan; + + GroupId rootGroupId = GroupId.createGenerator().getNextId(); + Group rootGroup = new Group(rootGroupId, physicalPlan.getLogicalProperties()); + rootGroupRef.set(rootGroup); + } + + @Override + public Group getRoot() { + if (!fastInsertIntoValues) { + return super.getRoot(); + } + return rootGroupRef.get(); + } + + @Override + protected PhysicalPlan chooseNthPlan( + Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) { + if (!fastInsertIntoValues) { + return super.chooseNthPlan(rootGroup, physicalProperties, nthPlan); + } + return super.physicalPlan; + } + + @Override + protected PhysicalPlan postProcess(PhysicalPlan physicalPlan) { + if (!fastInsertIntoValues) { + return super.postProcess(physicalPlan); + } + return physicalPlan; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index 8ca5e0982a1d2e..2e7d9d2eebd93b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; import org.apache.doris.common.profile.ProfileManager.ProfileType; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.datasource.hive.HMSExternalTable; @@ -32,10 +33,13 @@ import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.load.loadv2.LoadStatistic; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; @@ -57,10 +61,12 @@ import org.apache.doris.planner.DataSink; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContext.ConnectType; +import org.apache.doris.qe.Coordinator; import org.apache.doris.qe.StmtExecutor; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; +import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -68,6 +74,8 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; /** * insert into select command implementation @@ -81,8 +89,8 @@ public class InsertIntoTableCommand extends Command implements ForwardWithSync, public static final Logger LOG = LogManager.getLogger(InsertIntoTableCommand.class); - private LogicalPlan originalLogicalQuery; - private LogicalPlan logicalQuery; + private LogicalPlan originLogicalQuery; + private Optional logicalQuery; private Optional labelName; /** * When source it's from job scheduler,it will be set. @@ -97,15 +105,15 @@ public class InsertIntoTableCommand extends Command implements ForwardWithSync, public InsertIntoTableCommand(LogicalPlan logicalQuery, Optional labelName, Optional insertCtx, Optional cte) { super(PlanType.INSERT_INTO_TABLE_COMMAND); - this.originalLogicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); - this.logicalQuery = originalLogicalQuery; + this.originLogicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); this.labelName = Objects.requireNonNull(labelName, "labelName should not be null"); + this.logicalQuery = Optional.empty(); this.insertCtx = insertCtx; this.cte = cte; } public LogicalPlan getLogicalQuery() { - return logicalQuery; + return logicalQuery.orElse(originLogicalQuery); } public Optional getLabelName() { @@ -143,9 +151,10 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor * For external uses such as creating a job, only basic analysis is needed without starting a transaction, * in which case this can be set to false. */ - public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor, + public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor stmtExecutor, boolean needBeginTransaction) throws Exception { - List qualifiedTargetTableName = InsertUtils.getTargetTableQualified(logicalQuery, ctx); + List qualifiedTargetTableName = InsertUtils.getTargetTableQualified(originLogicalQuery, ctx); + AbstractInsertExecutor insertExecutor; int retryTimes = 0; while (++retryTimes < Math.max(ctx.getSessionVariable().dmlPlanRetryTimes, 3)) { @@ -161,7 +170,7 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor } BuildInsertExecutorResult buildResult; try { - buildResult = initPlanOnce(ctx, executor, targetTableIf); + buildResult = initPlanOnce(ctx, stmtExecutor, targetTableIf); } catch (Throwable e) { Throwables.throwIfInstanceOf(e, RuntimeException.class); throw new IllegalStateException(e.getMessage(), e); @@ -170,6 +179,7 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor if (!needBeginTransaction) { return insertExecutor; } + // lock after plan and check does table's schema changed to ensure we lock table order by id. TableIf newestTargetTableIf = RelationUtil.getTable(qualifiedTargetTableName, ctx.getEnv()); newestTargetTableIf.readLock(); @@ -206,13 +216,11 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor Throwables.throwIfInstanceOf(e, RuntimeException.class); throw new IllegalStateException(e.getMessage(), e); } - executor.setProfileType(ProfileType.LOAD); + stmtExecutor.setProfileType(ProfileType.LOAD); // We exposed @StmtExecutor#cancel as a unified entry point for statement interruption, // so we need to set this here insertExecutor.getCoordinator().setTxnId(insertExecutor.getTxnId()); - executor.setCoord(insertExecutor.getCoordinator()); - // for prepare and execute, avoiding normalization for every execute command - this.originalLogicalQuery = this.logicalQuery; + stmtExecutor.setCoord(insertExecutor.getCoordinator()); return insertExecutor; } LOG.warn("insert plan failed {} times. query id is {}.", retryTimes, DebugUtil.printId(ctx.queryId())); @@ -220,88 +228,156 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor } private BuildInsertExecutorResult initPlanOnce(ConnectContext ctx, - StmtExecutor executor, TableIf targetTableIf) throws Throwable { - AbstractInsertExecutor insertExecutor; - // 1. process inline table (default values, empty values) - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, insertCtx); - if (cte.isPresent()) { - this.logicalQuery = ((LogicalPlan) cte.get().withChildren(logicalQuery)); - } - OlapGroupCommitInsertExecutor.analyzeGroupCommit(ctx, targetTableIf, this.logicalQuery, this.insertCtx); - LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext()); - NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); - executor.setPlanner(planner); - executor.checkBlockRules(); - if (ctx.getConnectType() == ConnectType.MYSQL && ctx.getMysqlChannel() != null) { - ctx.getMysqlChannel().reset(); - } - Optional> plan = (planner.getPhysicalPlan() - .>collect(PhysicalSink.class::isInstance)).stream() - .findAny(); - Preconditions.checkArgument(plan.isPresent(), "insert into command must contain target table"); - PhysicalSink physicalSink = plan.get(); - DataSink sink = planner.getFragments().get(0).getSink(); - // Transaction insert should reuse the label in the transaction. - String label = this.labelName.orElse( - ctx.isTxnModel() ? null : String.format("label_%x_%x", ctx.queryId().hi, ctx.queryId().lo)); - - if (physicalSink instanceof PhysicalOlapTableSink) { - boolean emptyInsert = childIsEmptyRelation(physicalSink); - OlapTable olapTable = (OlapTable) targetTableIf; - // the insertCtx contains some variables to adjust SinkNode - if (ctx.isTxnModel()) { - insertExecutor = new OlapTxnInsertExecutor(ctx, olapTable, label, planner, insertCtx, emptyInsert); - } else if (ctx.isGroupCommit()) { - insertExecutor = new OlapGroupCommitInsertExecutor(ctx, olapTable, label, planner, insertCtx, - emptyInsert); - } else { - insertExecutor = new OlapInsertExecutor(ctx, olapTable, label, planner, insertCtx, emptyInsert); + StmtExecutor stmtExecutor, TableIf targetTableIf) throws Throwable { + targetTableIf.readLock(); + try { + Optional analyzeContext = Optional.of( + CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY) + ); + // process inline table (default values, empty values) + this.logicalQuery = Optional.of((LogicalPlan) InsertUtils.normalizePlan( + originLogicalQuery, targetTableIf, analyzeContext, insertCtx + )); + if (cte.isPresent()) { + this.logicalQuery = Optional.of((LogicalPlan) cte.get().withChildren(logicalQuery.get())); } + OlapGroupCommitInsertExecutor.analyzeGroupCommit( + ctx, targetTableIf, this.logicalQuery.get(), this.insertCtx); + } finally { + targetTableIf.readUnlock(); + } + + LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery.get(), ctx.getStatementContext()); + return planInsertExecutor(ctx, stmtExecutor, logicalPlanAdapter, targetTableIf); + } - boolean isEnableMemtableOnSinkNode = - olapTable.getTableProperty().getUseSchemaLightChange() - ? insertExecutor.getCoordinator().getQueryOptions().isEnableMemtableOnSinkNode() - : false; - insertExecutor.getCoordinator().getQueryOptions() - .setEnableMemtableOnSinkNode(isEnableMemtableOnSinkNode); - } else if (physicalSink instanceof PhysicalHiveTableSink) { - boolean emptyInsert = childIsEmptyRelation(physicalSink); - HMSExternalTable hiveExternalTable = (HMSExternalTable) targetTableIf; - if (hiveExternalTable.isHiveTransactionalTable()) { - throw new AnalysisException("Not supported insert into hive transactional table."); + // we should select the factory type first, but we can not initial InsertExecutor at this time, + // because Nereids's DistributePlan are not gernerated, so we return factory and after the + // DistributePlan have been generated, we can create InsertExecutor + private ExecutorFactory selectInsertExecutorFactory( + NereidsPlanner planner, ConnectContext ctx, StmtExecutor stmtExecutor, TableIf targetTableIf) { + try { + stmtExecutor.setPlanner(planner); + stmtExecutor.checkBlockRules(); + if (ctx.getConnectType() == ConnectType.MYSQL && ctx.getMysqlChannel() != null) { + ctx.getMysqlChannel().reset(); } - insertExecutor = new HiveInsertExecutor(ctx, hiveExternalTable, label, planner, - Optional.of(insertCtx.orElse((new HiveInsertCommandContext()))), emptyInsert); - // set hive query options - } else if (physicalSink instanceof PhysicalIcebergTableSink) { - boolean emptyInsert = childIsEmptyRelation(physicalSink); - IcebergExternalTable icebergExternalTable = (IcebergExternalTable) targetTableIf; - insertExecutor = new IcebergInsertExecutor(ctx, icebergExternalTable, label, planner, - Optional.of(insertCtx.orElse((new BaseExternalTableInsertCommandContext()))), emptyInsert); - } else if (physicalSink instanceof PhysicalJdbcTableSink) { - boolean emptyInsert = childIsEmptyRelation(physicalSink); - List cols = ((PhysicalJdbcTableSink) physicalSink).getCols(); - List slots = ((PhysicalJdbcTableSink) physicalSink).getOutput(); - if (physicalSink.children().size() == 1) { - if (physicalSink.child(0) instanceof PhysicalOneRowRelation - || physicalSink.child(0) instanceof PhysicalUnion) { - for (int i = 0; i < cols.size(); i++) { - if (!(cols.get(i).isAllowNull()) && slots.get(i).nullable()) { - throw new AnalysisException("Column `" + cols.get(i).getName() - + "` is not nullable, but the inserted value is nullable."); + Optional> plan = (planner.getPhysicalPlan() + .>collect(PhysicalSink.class::isInstance)).stream() + .findAny(); + Preconditions.checkArgument(plan.isPresent(), "insert into command must contain target table"); + PhysicalSink physicalSink = plan.get(); + DataSink dataSink = planner.getFragments().get(0).getSink(); + // Transaction insert should reuse the label in the transaction. + String label = this.labelName.orElse( + ctx.isTxnModel() ? null : String.format("label_%x_%x", ctx.queryId().hi, ctx.queryId().lo)); + + if (physicalSink instanceof PhysicalOlapTableSink) { + boolean emptyInsert = childIsEmptyRelation(physicalSink); + OlapTable olapTable = (OlapTable) targetTableIf; + + ExecutorFactory executorFactory; + // the insertCtx contains some variables to adjust SinkNode + if (ctx.isTxnModel()) { + executorFactory = ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new OlapTxnInsertExecutor(ctx, olapTable, label, planner, insertCtx, emptyInsert) + ); + } else if (ctx.isGroupCommit()) { + executorFactory = ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new OlapGroupCommitInsertExecutor( + ctx, olapTable, label, planner, insertCtx, emptyInsert + ) + ); + } else { + executorFactory = ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new OlapInsertExecutor(ctx, olapTable, label, planner, insertCtx, emptyInsert) + ); + } + + return executorFactory.onCreate(executor -> { + Coordinator coordinator = executor.getCoordinator(); + boolean isEnableMemtableOnSinkNode = olapTable.getTableProperty().getUseSchemaLightChange() + && coordinator.getQueryOptions().isEnableMemtableOnSinkNode(); + coordinator.getQueryOptions().setEnableMemtableOnSinkNode(isEnableMemtableOnSinkNode); + }); + } else if (physicalSink instanceof PhysicalHiveTableSink) { + boolean emptyInsert = childIsEmptyRelation(physicalSink); + HMSExternalTable hiveExternalTable = (HMSExternalTable) targetTableIf; + if (hiveExternalTable.isHiveTransactionalTable()) { + throw new UserException("Not supported insert into hive transactional table."); + } + + return ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new HiveInsertExecutor(ctx, hiveExternalTable, label, planner, + Optional.of(insertCtx.orElse((new HiveInsertCommandContext()))), emptyInsert) + ); + // set hive query options + } else if (physicalSink instanceof PhysicalIcebergTableSink) { + boolean emptyInsert = childIsEmptyRelation(physicalSink); + IcebergExternalTable icebergExternalTable = (IcebergExternalTable) targetTableIf; + return ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new IcebergInsertExecutor(ctx, icebergExternalTable, label, planner, + Optional.of(insertCtx.orElse((new BaseExternalTableInsertCommandContext()))), + emptyInsert + ) + ); + } else if (physicalSink instanceof PhysicalJdbcTableSink) { + boolean emptyInsert = childIsEmptyRelation(physicalSink); + List cols = ((PhysicalJdbcTableSink) physicalSink).getCols(); + List slots = physicalSink.getOutput(); + if (physicalSink.children().size() == 1) { + if (physicalSink.child(0) instanceof PhysicalOneRowRelation + || physicalSink.child(0) instanceof PhysicalUnion) { + for (int i = 0; i < cols.size(); i++) { + if (!(cols.get(i).isAllowNull()) && slots.get(i).nullable()) { + throw new AnalysisException("Column `" + cols.get(i).getName() + + "` is not nullable, but the inserted value is nullable."); + } } } } + JdbcExternalTable jdbcExternalTable = (JdbcExternalTable) targetTableIf; + return ExecutorFactory.from( + planner, + dataSink, + physicalSink, + () -> new JdbcInsertExecutor(ctx, jdbcExternalTable, label, planner, + Optional.of(insertCtx.orElse((new JdbcInsertCommandContext()))), emptyInsert) + ); + } else { + // TODO: support other table types + throw new AnalysisException("insert into command only support [olap, hive, iceberg, jdbc] table"); } - JdbcExternalTable jdbcExternalTable = (JdbcExternalTable) targetTableIf; - insertExecutor = new JdbcInsertExecutor(ctx, jdbcExternalTable, label, planner, - Optional.of(insertCtx.orElse((new JdbcInsertCommandContext()))), emptyInsert); - } else { - // TODO: support other table types - throw new AnalysisException("insert into command only support [olap, hive, iceberg, jdbc] table"); + } catch (Throwable t) { + Throwables.propagateIfInstanceOf(t, RuntimeException.class); + throw new IllegalStateException(t.getMessage(), t); } - return new BuildInsertExecutorResult(planner, insertExecutor, sink, physicalSink); + } + + private BuildInsertExecutorResult planInsertExecutor( + ConnectContext ctx, StmtExecutor stmtExecutor, + LogicalPlanAdapter logicalPlanAdapter, TableIf targetTableIf) throws Throwable { + LogicalPlan logicalPlan = logicalPlanAdapter.getLogicalPlan(); + boolean supportFastInsertIntoValues = InsertUtils.supportFastInsertIntoValues(logicalPlan, targetTableIf, ctx); + FastInsertIntoValuesPlanner planner = new FastInsertIntoValuesPlanner( + ctx.getStatementContext(), supportFastInsertIntoValues); + planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); + return selectInsertExecutorFactory(planner, ctx, stmtExecutor, targetTableIf).build(); } private void runInternal(ConnectContext ctx, StmtExecutor executor) throws Exception { @@ -314,14 +390,14 @@ private void runInternal(ConnectContext ctx, StmtExecutor executor) throws Excep } public boolean isExternalTableSink() { - return !(logicalQuery instanceof UnboundTableSink); + return !(getLogicalQuery() instanceof UnboundTableSink); } /** * get the target table of the insert command */ public TableIf getTable(ConnectContext ctx) throws Exception { - TableIf targetTableIf = InsertUtils.getTargetTable(originalLogicalQuery, ctx); + TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, ctx); if (!Env.getCurrentEnv().getAccessManager() .checkTblPriv(ConnectContext.get(), targetTableIf.getDatabase().getCatalog().getName(), targetTableIf.getDatabase().getFullName(), targetTableIf.getName(), @@ -337,25 +413,37 @@ public TableIf getTable(ConnectContext ctx) throws Exception { * get the target columns of the insert command */ public List getTargetColumns() { - if (originalLogicalQuery instanceof UnboundTableSink) { + if (originLogicalQuery instanceof UnboundTableSink) { UnboundLogicalSink unboundTableSink - = (UnboundTableSink) originalLogicalQuery; + = (UnboundTableSink) originLogicalQuery; return CollectionUtils.isEmpty(unboundTableSink.getColNames()) ? null : unboundTableSink.getColNames(); } else { throw new AnalysisException( - "the root of plan should be [UnboundTableSink], but it is " + originalLogicalQuery.getType()); + "the root of plan should be [UnboundTableSink], but it is " + originLogicalQuery.getType()); } } @Override public Plan getExplainPlan(ConnectContext ctx) { - Plan plan = InsertUtils.getPlanForExplain(ctx, this.logicalQuery); + Optional analyzeContext = Optional.of( + CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY) + ); + Plan plan = InsertUtils.getPlanForExplain(ctx, analyzeContext, getLogicalQuery()); if (cte.isPresent()) { plan = cte.get().withChildren(plan); } return plan; } + @Override + public Optional getExplainPlanner(LogicalPlan logicalPlan, StatementContext ctx) { + ConnectContext connectContext = ctx.getConnectContext(); + TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, connectContext); + boolean supportFastInsertIntoValues + = InsertUtils.supportFastInsertIntoValues(logicalPlan, targetTableIf, connectContext); + return Optional.of(new FastInsertIntoValuesPlanner(ctx, supportFastInsertIntoValues)); + } + @Override public R accept(PlanVisitor visitor, C context) { return visitor.visitInsertIntoTableCommand(this, context); @@ -383,6 +471,46 @@ public RedirectStatus toRedirectStatus() { } } + /** + * this factory is used to delay create the AbstractInsertExecutor until the DistributePlan is generated + * by NereidsPlanner + */ + private static class ExecutorFactory { + public final NereidsPlanner planner; + public final DataSink dataSink; + public final PhysicalSink physicalSink; + public final Supplier executorSupplier; + private List> createCallback; + + private ExecutorFactory(NereidsPlanner planner, DataSink dataSink, PhysicalSink physicalSink, + Supplier executorSupplier) { + this.planner = planner; + this.dataSink = dataSink; + this.physicalSink = physicalSink; + this.executorSupplier = executorSupplier; + this.createCallback = Lists.newArrayList(); + } + + public static ExecutorFactory from( + NereidsPlanner planner, DataSink dataSink, PhysicalSink physicalSink, + Supplier executorSupplier) { + return new ExecutorFactory(planner, dataSink, physicalSink, executorSupplier); + } + + public ExecutorFactory onCreate(Consumer onCreate) { + this.createCallback.add(onCreate); + return this; + } + + public BuildInsertExecutorResult build() { + AbstractInsertExecutor executor = executorSupplier.get(); + for (Consumer callback : createCallback) { + callback.accept(executor); + } + return new BuildInsertExecutorResult(planner, executor, dataSink, physicalSink); + } + } + private static class BuildInsertExecutorResult { private final NereidsPlanner planner; private final AbstractInsertExecutor executor; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoValuesAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoValuesAnalyzer.java new file mode 100644 index 00000000000000..1c630a41c846fe --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoValuesAnalyzer.java @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.jobs.executor.AbstractBatchJobExecutor; +import org.apache.doris.nereids.jobs.rewrite.RewriteJob; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.analysis.BindSink; +import org.apache.doris.nereids.rules.expression.ExpressionRewrite; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteRule; +import org.apache.doris.nereids.rules.expression.rules.ConvertAggStateCast; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; +import org.apache.doris.nereids.rules.rewrite.MergeProjects; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation; +import org.apache.doris.nereids.rules.rewrite.PushProjectIntoUnion; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; +import org.apache.doris.nereids.types.DataType; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import java.util.List; + +/** InsertIntoValuesAnalyzer */ +public class InsertIntoValuesAnalyzer extends AbstractBatchJobExecutor { + public static final List INSERT_JOBS = jobs( + bottomUp( + new InlineTableToUnionOrOneRowRelation(), + new BindSink(), + new MergeProjects(), + // after bind olap table sink, the LogicalProject will be generated under LogicalOlapTableSink, + // we should convert the agg state function in the project, and evaluate some env parameters + // like encrypt key reference, for example: `values (aes_encrypt("abc",key test.my_key))`, + // we should replace the `test.my_key` to real key + new RewriteInsertIntoExpressions(ExpressionRewrite.bottomUp( + ConvertAggStateCast.INSTANCE, + FoldConstantRuleOnFE.PATTERN_MATCH_INSTANCE + )) + ) + ); + + public static final List BATCH_INSERT_JOBS = jobs( + bottomUp( + new InlineTableToUnionOrOneRowRelation(), + new BindSink(), + new MergeProjects(), + + // the BatchInsertIntoTableCommand need send StringLiteral to backend, + // and only support alias(literal as xx) or alias(cast(literal as xx)), + // but not support alias(cast(slotRef as xx)) which create in BindSink, + // we should push down the cast into Union or OneRowRelation. + // the InsertIntoTableCommand support translate slotRef in the TPlan, + // so we don't need this rules, just evaluate in backend + new PushProjectIntoUnion(), + new PushProjectIntoOneRowRelation(), + + new RewriteBatchInsertIntoExpressions(ExpressionRewrite.bottomUp( + ConvertAggStateCast.INSTANCE, + FoldConstantRuleOnFE.PATTERN_MATCH_INSTANCE + )) + ) + ); + + private final boolean batchInsert; + + public InsertIntoValuesAnalyzer(CascadesContext cascadesContext, boolean batchInsert) { + super(cascadesContext); + this.batchInsert = batchInsert; + } + + @Override + public List getJobs() { + return batchInsert ? BATCH_INSERT_JOBS : INSERT_JOBS; + } + + // we only rewrite the project's expression + private static class RewriteInsertIntoExpressions extends ExpressionRewrite { + public RewriteInsertIntoExpressions(ExpressionRewriteRule... rules) { + super(rules); + } + + @Override + public List buildRules() { + return ImmutableList.of( + new ProjectExpressionRewrite().build() + ); + } + } + + // we only rewrite the project's and one row relation expression + private static class RewriteBatchInsertIntoExpressions extends ExpressionRewrite { + public RewriteBatchInsertIntoExpressions(ExpressionRewriteRule... rules) { + super(rules); + } + + @Override + public List buildRules() { + return ImmutableList.of( + new ProjectExpressionRewrite().build(), + new OneRowRelationExpressionRewrite().build() + ); + } + } + + private static class InlineTableToUnionOrOneRowRelation extends OneRewriteRuleFactory { + @Override + public Rule build() { + return inlineTable().then(inlineTable -> { + List> originConstants = inlineTable.getConstantExprsList(); + if (originConstants.size() > 1) { + Pair>, List> castedConstantsAndNullables + = LogicalUnion.castCommonDataTypeAndNullableByConstants(originConstants); + List> castedRows = castedConstantsAndNullables.key(); + List nullables = castedConstantsAndNullables.value(); + List outputs = Lists.newArrayList(); + List firstRow = originConstants.get(0); + for (int columnId = 0; columnId < firstRow.size(); columnId++) { + String name = firstRow.get(columnId).getName(); + DataType commonDataType = castedRows.get(0).get(columnId).getDataType(); + outputs.add(new SlotReference(name, commonDataType, nullables.get(columnId))); + } + return new LogicalUnion(Qualifier.ALL, castedRows, ImmutableList.of()).withNewOutputs(outputs); + } else if (originConstants.size() == 1) { + return new LogicalOneRowRelation(StatementScopeIdGenerator.newRelationId(), originConstants.get(0)); + } else { + throw new AnalysisException("Illegal inline table with empty constants"); + } + }).toRule(RuleType.LOGICAL_INLINE_TABLE_TO_LOGICAL_UNION_OR_ONE_ROW_RELATION); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java index c4db84f378bd97..72e5c4c14e3b0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java @@ -32,13 +32,16 @@ import org.apache.doris.insertoverwrite.InsertOverwriteUtil; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; @@ -82,7 +85,8 @@ public class InsertOverwriteTableCommand extends Command implements ForwardWithS private static final Logger LOG = LogManager.getLogger(InsertOverwriteTableCommand.class); - private LogicalPlan logicalQuery; + private LogicalPlan originLogicalQuery; + private Optional logicalQuery; private Optional labelName; private final Optional cte; private AtomicBoolean isCancelled = new AtomicBoolean(false); @@ -94,7 +98,8 @@ public class InsertOverwriteTableCommand extends Command implements ForwardWithS public InsertOverwriteTableCommand(LogicalPlan logicalQuery, Optional labelName, Optional cte) { super(PlanType.INSERT_INTO_TABLE_COMMAND); - this.logicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); + this.originLogicalQuery = Objects.requireNonNull(logicalQuery, "logicalQuery should not be null"); + this.logicalQuery = Optional.empty(); this.labelName = Objects.requireNonNull(labelName, "labelName should not be null"); this.cte = cte; } @@ -103,14 +108,18 @@ public void setLabelName(Optional labelName) { this.labelName = labelName; } - public boolean isAutoDetectOverwrite() { + public boolean isAutoDetectOverwrite(LogicalPlan logicalQuery) { return (logicalQuery instanceof UnboundTableSink) - && ((UnboundTableSink) this.logicalQuery).isAutoDetectPartition(); + && ((UnboundTableSink) logicalQuery).isAutoDetectPartition(); + } + + public LogicalPlan getLogicalQuery() { + return logicalQuery.orElse(originLogicalQuery); } @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { - TableIf targetTableIf = InsertUtils.getTargetTable(logicalQuery, ctx); + TableIf targetTableIf = InsertUtils.getTargetTable(originLogicalQuery, ctx); //check allow insert overwrite if (!allowInsertOverwrite(targetTableIf)) { String errMsg = "insert into overwrite only support OLAP and HMS/ICEBERG table." @@ -122,12 +131,20 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { if (targetTableIf instanceof MTMV && !MTMVUtil.allowModifyMTMVData(ctx)) { throw new AnalysisException("Not allowed to perform current operation on async materialized view"); } - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, Optional.empty()); + Optional analyzeContext = Optional.of( + CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY) + ); + this.logicalQuery = Optional.of((LogicalPlan) InsertUtils.normalizePlan( + originLogicalQuery, targetTableIf, analyzeContext, Optional.empty())); if (cte.isPresent()) { - this.logicalQuery = (LogicalPlan) logicalQuery.withChildren(cte.get().withChildren( - this.logicalQuery.child(0))); + LogicalPlan logicalQuery = this.logicalQuery.get(); + this.logicalQuery = Optional.of( + (LogicalPlan) logicalQuery.withChildren( + cte.get().withChildren(logicalQuery.child(0)) + ) + ); } - + LogicalPlan logicalQuery = this.logicalQuery.get(); LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext()); NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); @@ -172,7 +189,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { isRunning.set(true); long taskId = 0; try { - if (isAutoDetectOverwrite()) { + if (isAutoDetectOverwrite(getLogicalQuery())) { // taskId here is a group id. it contains all replace tasks made and registered in rpc process. taskId = insertOverwriteManager.registerTaskGroup(); // When inserting, BE will call to replace partition by FrontendService. FE will register new temp @@ -220,7 +237,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { } } catch (Exception e) { LOG.warn("insert into overwrite failed with task(or group) id " + taskId); - if (isAutoDetectOverwrite()) { + if (isAutoDetectOverwrite(getLogicalQuery())) { insertOverwriteManager.taskGroupFail(taskId); } else { insertOverwriteManager.taskFail(taskId); @@ -288,6 +305,7 @@ private void insertIntoPartitions(ConnectContext ctx, StmtExecutor executor, Lis // copy sink tot replace by tempPartitions UnboundLogicalSink copySink; InsertCommandContext insertCtx; + LogicalPlan logicalQuery = getLogicalQuery(); if (logicalQuery instanceof UnboundTableSink) { UnboundTableSink sink = (UnboundTableSink) logicalQuery; copySink = (UnboundLogicalSink) UnboundTableSinkCreator.createUnboundTableSink( @@ -343,6 +361,7 @@ private void insertIntoPartitions(ConnectContext ctx, StmtExecutor executor, Lis */ private void insertIntoAutoDetect(ConnectContext ctx, StmtExecutor executor, long groupId) throws Exception { InsertCommandContext insertCtx; + LogicalPlan logicalQuery = getLogicalQuery(); if (logicalQuery instanceof UnboundTableSink) { // 1. when overwrite auto-detect, allow auto partition or not is controlled by session variable. // 2. we save and pass overwrite auto detect by insertCtx @@ -363,7 +382,23 @@ private void insertIntoAutoDetect(ConnectContext ctx, StmtExecutor executor, lon @Override public Plan getExplainPlan(ConnectContext ctx) { - return InsertUtils.getPlanForExplain(ctx, this.logicalQuery); + Optional analyzeContext = Optional.of( + CascadesContext.initContext(ctx.getStatementContext(), originLogicalQuery, PhysicalProperties.ANY) + ); + return InsertUtils.getPlanForExplain(ctx, analyzeContext, getLogicalQuery()); + } + + @Override + public Optional getExplainPlanner(LogicalPlan logicalPlan, StatementContext ctx) { + LogicalPlan logicalQuery = getLogicalQuery(); + if (logicalQuery instanceof UnboundTableSink) { + boolean allowAutoPartition = ctx.getConnectContext().getSessionVariable().isEnableAutoCreateWhenOverwrite(); + OlapInsertCommandContext insertCtx = new OlapInsertCommandContext(allowAutoPartition, true); + InsertIntoTableCommand insertIntoTableCommand = new InsertIntoTableCommand( + logicalQuery, labelName, Optional.of(insertCtx), Optional.empty()); + return insertIntoTableCommand.getExplainPlanner(logicalPlan, ctx); + } + return Optional.empty(); } public boolean isForceDropPartition() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 9b8314abbedfe4..e3ea8e8ba2b457 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -29,29 +29,39 @@ import org.apache.doris.common.FormatOptions; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.jdbc.JdbcExternalTable; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.analyzer.UnboundAlias; +import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; +import org.apache.doris.nereids.analyzer.UnboundInlineTable; import org.apache.doris.nereids.analyzer.UnboundJdbcTableSink; -import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.analyzer.UnboundStar; import org.apache.doris.nereids.analyzer.UnboundTableSink; +import org.apache.doris.nereids.analyzer.UnboundVariable; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.parser.LogicalPlanBuilder; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.expression.rules.ConvertAggStateCast; +import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.DefaultValueSlot; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; +import org.apache.doris.nereids.types.AggStateType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.RelationUtil; import org.apache.doris.nereids.util.TypeCoercionUtils; @@ -78,7 +88,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; @@ -250,16 +262,19 @@ private static void beginBatchInsertTransaction(ConnectContext ctx, /** * normalize plan to let it could be process correctly by nereids */ - public static Plan normalizePlan(LogicalPlan plan, TableIf table, Optional insertCtx) { + public static Plan normalizePlan(LogicalPlan plan, TableIf table, + Optional analyzeContext, + Optional insertCtx) { table.readLock(); try { - return normalizePlanWithoutLock(plan, table, insertCtx); + return normalizePlanWithoutLock(plan, table, analyzeContext, insertCtx); } finally { table.readUnlock(); } } private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, + Optional analyzeContext, Optional insertCtx) { UnboundLogicalSink unboundLogicalSink = (UnboundLogicalSink) plan; if (table instanceof HMSExternalTable) { @@ -334,21 +349,39 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, } Plan query = unboundLogicalSink.child(); checkGeneratedColumnForInsertIntoSelect(table, unboundLogicalSink, insertCtx); - if (!(query instanceof LogicalInlineTable)) { + if (!(query instanceof UnboundInlineTable)) { return plan; } - LogicalInlineTable logicalInlineTable = (LogicalInlineTable) query; - ImmutableList.Builder oneRowRelationBuilder = ImmutableList.builder(); + + UnboundInlineTable unboundInlineTable = (UnboundInlineTable) query; + ImmutableList.Builder> optimizedRowConstructors + = ImmutableList.builderWithExpectedSize(unboundInlineTable.getConstantExprsList().size()); List columns = table.getBaseSchema(false); - for (List values : logicalInlineTable.getConstantExprsList()) { - ImmutableList.Builder constantExprs = ImmutableList.builder(); + ConnectContext context = ConnectContext.get(); + ExpressionRewriteContext rewriteContext = null; + if (context != null && context.getStatementContext() != null) { + rewriteContext = new ExpressionRewriteContext( + CascadesContext.initContext( + context.getStatementContext(), unboundInlineTable, PhysicalProperties.ANY + ) + ); + } + + Optional analyzer = analyzeContext.map( + cascadesContext -> buildExprAnalyzer(plan, cascadesContext) + ); + + for (List values : unboundInlineTable.getConstantExprsList()) { + ImmutableList.Builder optimizedRowConstructor = ImmutableList.builder(); if (values.isEmpty()) { if (CollectionUtils.isNotEmpty(unboundLogicalSink.getColNames())) { throw new AnalysisException("value list should not be empty if columns are specified"); } - for (Column column : columns) { - constantExprs.add(generateDefaultExpression(column)); + for (int i = 0; i < columns.size(); i++) { + Column column = columns.get(i); + NamedExpression defaultExpression = generateDefaultExpression(column); + addColumnValue(analyzer, optimizedRowConstructor, defaultExpression); } } else { if (CollectionUtils.isNotEmpty(unboundLogicalSink.getColNames())) { @@ -374,10 +407,15 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, + "' in table '" + table.getName() + "' is not allowed."); } if (values.get(i) instanceof DefaultValueSlot) { - constantExprs.add(generateDefaultExpression(sameNameColumn)); + NamedExpression defaultExpression = generateDefaultExpression(sameNameColumn); + addColumnValue(analyzer, optimizedRowConstructor, defaultExpression); } else { DataType targetType = DataType.fromCatalogType(sameNameColumn.getType()); - constantExprs.add((NamedExpression) castValue(values.get(i), targetType)); + Expression castValue = castValue(values.get(i), targetType); + castValue = rewriteContext == null + ? castValue + : FoldConstantRuleOnFE.evaluate(castValue, rewriteContext); + addColumnValue(analyzer, optimizedRowConstructor, (NamedExpression) castValue); } } } else { @@ -392,37 +430,114 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, + "' in table '" + table.getName() + "' is not allowed."); } if (values.get(i) instanceof DefaultValueSlot) { - constantExprs.add(generateDefaultExpression(columns.get(i))); + NamedExpression defaultExpression = generateDefaultExpression(columns.get(i)); + addColumnValue(analyzer, optimizedRowConstructor, defaultExpression); } else { DataType targetType = DataType.fromCatalogType(columns.get(i).getType()); - constantExprs.add((NamedExpression) castValue(values.get(i), targetType)); + Expression castValue = castValue(values.get(i), targetType); + castValue = rewriteContext == null + ? castValue + : FoldConstantRuleOnFE.evaluate(castValue, rewriteContext); + addColumnValue(analyzer, optimizedRowConstructor, (NamedExpression) castValue); } } } } - oneRowRelationBuilder.add(new UnboundOneRowRelation( - StatementScopeIdGenerator.newRelationId(), constantExprs.build())); + optimizedRowConstructors.add(optimizedRowConstructor.build()); } - List oneRowRelations = oneRowRelationBuilder.build(); - if (oneRowRelations.size() == 1) { - return plan.withChildren(oneRowRelations.get(0)); - } else { - return plan.withChildren( - LogicalPlanBuilder.reduceToLogicalPlanTree(0, oneRowRelations.size() - 1, - oneRowRelations, Qualifier.ALL)); + return plan.withChildren(new LogicalInlineTable(optimizedRowConstructors.build())); + } + + /** buildAnalyzer */ + public static ExpressionAnalyzer buildExprAnalyzer(Plan plan, CascadesContext analyzeContext) { + return new ExpressionAnalyzer(plan, new Scope(ImmutableList.of()), + analyzeContext, false, false) { + @Override + public Expression visitCast(Cast cast, ExpressionRewriteContext context) { + Expression expr = super.visitCast(cast, context); + if (expr instanceof Cast) { + if (expr.child(0).getDataType() instanceof AggStateType) { + expr = ConvertAggStateCast.convert((Cast) expr); + } else { + expr = FoldConstantRuleOnFE.evaluate(expr, context); + } + } + return expr; + } + + @Override + public Expression visitUnboundFunction(UnboundFunction unboundFunction, ExpressionRewriteContext context) { + Expression expr = super.visitUnboundFunction(unboundFunction, context); + if (expr instanceof UnboundFunction) { + throw new IllegalStateException("Can not analyze function " + unboundFunction.getName()); + } + return expr; + } + + @Override + public Expression visitUnboundSlot(UnboundSlot unboundSlot, ExpressionRewriteContext context) { + Expression expr = super.visitUnboundSlot(unboundSlot, context); + if (expr instanceof UnboundFunction) { + throw new AnalysisException("Can not analyze slot " + unboundSlot.getName()); + } + return expr; + } + + @Override + public Expression visitUnboundVariable(UnboundVariable unboundVariable, ExpressionRewriteContext context) { + Expression expr = super.visitUnboundVariable(unboundVariable, context); + if (expr instanceof UnboundVariable) { + throw new AnalysisException("Can not analyze variable " + unboundVariable.getName()); + } + return expr; + } + + @Override + public Expression visitUnboundAlias(UnboundAlias unboundAlias, ExpressionRewriteContext context) { + Expression expr = super.visitUnboundAlias(unboundAlias, context); + if (expr instanceof UnboundVariable) { + throw new AnalysisException("Can not analyze alias"); + } + return expr; + } + + @Override + public Expression visitUnboundStar(UnboundStar unboundStar, ExpressionRewriteContext context) { + Expression expr = super.visitUnboundStar(unboundStar, context); + if (expr instanceof UnboundStar) { + List qualifier = unboundStar.getQualifier(); + List qualified = new ArrayList<>(qualifier); + qualified.add("*"); + throw new AnalysisException("Can not analyze " + StringUtils.join(qualified, ".")); + } + return expr; + } + }; + } + + private static void addColumnValue( + Optional analyzer, + ImmutableList.Builder optimizedRowConstructor, + NamedExpression value) { + if (analyzer.isPresent() && !(value instanceof Alias && value.child(0) instanceof Literal)) { + ExpressionAnalyzer expressionAnalyzer = analyzer.get(); + value = (NamedExpression) expressionAnalyzer.analyze( + value, new ExpressionRewriteContext(expressionAnalyzer.getCascadesContext()) + ); } + optimizedRowConstructor.add(value); } - private static Expression castValue(Expression value, DataType targetType) { + private static Alias castValue(Expression value, DataType targetType) { if (value instanceof Alias) { Expression oldChild = value.child(0); Expression newChild = TypeCoercionUtils.castUnbound(oldChild, targetType); - return oldChild == newChild ? value : value.withChildren(newChild); + return (Alias) (oldChild == newChild ? value : value.withChildren(newChild)); } else if (value instanceof UnboundAlias) { UnboundAlias unboundAlias = (UnboundAlias) value; return new Alias(TypeCoercionUtils.castUnbound(unboundAlias.child(), targetType)); } else { - return TypeCoercionUtils.castUnbound(value, targetType); + return new Alias(TypeCoercionUtils.castUnbound(value, targetType)); } } @@ -489,8 +604,18 @@ private static NamedExpression generateDefaultExpression(Column column) { /** * get plan for explain. */ - public static Plan getPlanForExplain(ConnectContext ctx, LogicalPlan logicalQuery) { - return InsertUtils.normalizePlan(logicalQuery, InsertUtils.getTargetTable(logicalQuery, ctx), Optional.empty()); + public static Plan getPlanForExplain( + ConnectContext ctx, Optional analyzeContext, LogicalPlan logicalQuery) { + return InsertUtils.normalizePlan( + logicalQuery, InsertUtils.getTargetTable(logicalQuery, ctx), analyzeContext, Optional.empty()); + } + + /** supportFastInsertIntoValues */ + public static boolean supportFastInsertIntoValues( + LogicalPlan logicalPlan, TableIf targetTableIf, ConnectContext ctx) { + return logicalPlan instanceof UnboundTableSink && logicalPlan.child(0) instanceof InlineTable + && targetTableIf instanceof OlapTable + && ctx != null && ctx.getSessionVariable().isEnableFastAnalyzeInsertIntoValues(); } // check for insert into t1(a,b,gen_col) select 1,2,3; @@ -514,7 +639,7 @@ private static void checkGeneratedColumnForInsertIntoSelect(TableIf table, return; } Plan query = unboundLogicalSink.child(); - if (table instanceof OlapTable && !(query instanceof LogicalInlineTable)) { + if (table instanceof OlapTable && !(query instanceof InlineTable)) { OlapTable olapTable = (OlapTable) table; Set insertNames = Sets.newHashSet(); if (unboundLogicalSink.getColNames() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java index b90a616cd7fb3c..8d8ce33c6c846f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapGroupCommitInsertExecutor.java @@ -33,9 +33,9 @@ import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation; import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; -import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.planner.GroupCommitPlanner; @@ -134,9 +134,9 @@ protected static void analyzeGroupCommit(ConnectContext ctx, TableIf table, Logi && ((OlapInsertCommandContext) insertCtx.get()).isOverwrite()), () -> "is overwrite command")); Plan tableSinkChild = tableSink.child(); conditions.add(Pair.of( - () -> tableSinkChild instanceof OneRowRelation || (tableSinkChild instanceof LogicalUnion - && tableSinkChild.getExpressions().size() > 0) - || tableSinkChild instanceof LogicalInlineTable, + () -> tableSinkChild instanceof OneRowRelation + || (tableSinkChild instanceof LogicalUnion && tableSinkChild.getExpressions().size() > 0) + || tableSinkChild instanceof InlineTable, () -> "should be one row relation or union or inline table, class: " + tableSinkChild.getClass().getName() + (tableSinkChild instanceof LogicalUnion ? ", expression size is 0" : ""))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java index b2a2a1d83ca3e7..748bc8fdfa2223 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalInlineTable.java @@ -17,15 +17,19 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; @@ -36,7 +40,7 @@ /** * represent value list such as values(1), (2), (3) will generate LogicalInlineTable((1), (2), (3)). */ -public class LogicalInlineTable extends LogicalLeaf implements BlockFuncDepsPropagation { +public class LogicalInlineTable extends LogicalLeaf implements InlineTable, BlockFuncDepsPropagation { private final List> constantExprsList; @@ -44,11 +48,16 @@ public LogicalInlineTable(List> constantExprsList) { this(constantExprsList, Optional.empty(), Optional.empty()); } + /** LogicalInlineTable */ public LogicalInlineTable(List> constantExprsList, Optional groupExpression, Optional logicalProperties) { super(PlanType.LOGICAL_INLINE_TABLE, groupExpression, logicalProperties); - this.constantExprsList = ImmutableList.copyOf( + + if (constantExprsList.isEmpty()) { + throw new AnalysisException("constantExprsList should now be empty"); + } + this.constantExprsList = Utils.fastToImmutableList( Objects.requireNonNull(constantExprsList, "constantExprsList should not be null")); } @@ -63,23 +72,49 @@ public R accept(PlanVisitor visitor, C context) { @Override public List getExpressions() { - return constantExprsList.stream().flatMap(List::stream).collect(ImmutableList.toImmutableList()); + ImmutableList.Builder expressions = ImmutableList.builderWithExpectedSize( + constantExprsList.size() * constantExprsList.get(0).size()); + + for (List namedExpressions : constantExprsList) { + expressions.addAll(namedExpressions); + } + + return expressions.build(); } @Override public Plan withGroupExpression(Optional groupExpression) { - return null; + return new LogicalInlineTable( + constantExprsList, groupExpression, Optional.of(getLogicalProperties()) + ); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return null; + if (!children.isEmpty()) { + throw new AnalysisException("children should not be empty"); + } + return new LogicalInlineTable(constantExprsList, groupExpression, logicalProperties); } @Override public List computeOutput() { - return ImmutableList.of(); + int columnNum = constantExprsList.get(0).size(); + List firstRow = constantExprsList.get(0); + ImmutableList.Builder output = ImmutableList.builderWithExpectedSize(constantExprsList.size()); + for (int i = 0; i < columnNum; i++) { + NamedExpression firstRowColumn = firstRow.get(i); + boolean nullable = false; + for (List row : constantExprsList) { + if (row.get(i).nullable()) { + nullable = true; + break; + } + } + output.add(new SlotReference(firstRowColumn.getName(), firstRowColumn.getDataType(), nullable)); + } + return output.build(); } @Override @@ -98,4 +133,11 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(constantExprsList); } + + @Override + public String toString() { + return Utils.toSqlString("LogicalInlineTable[" + id.asInt() + "]", + "rowNum", constantExprsList.size(), + "constantExprsList", constantExprsList); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java index 9b78480c6a747e..7376cb10ba670a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java @@ -54,7 +54,7 @@ public LogicalOneRowRelation(RelationId relationId, List projec private LogicalOneRowRelation(RelationId relationId, List projects, Optional groupExpression, Optional logicalProperties) { super(relationId, PlanType.LOGICAL_ONE_ROW_RELATION, groupExpression, logicalProperties); - this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null")); + this.projects = Utils.fastToImmutableList(Objects.requireNonNull(projects, "projects can not be null")); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java index 2e4ddb55ff2f02..e13ec2864b3cd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSetOperation.java @@ -216,7 +216,8 @@ public int getArity() { return children.size(); } - private DataType getAssignmentCompatibleType(DataType left, DataType right) { + /** getAssignmentCompatibleType */ + public static DataType getAssignmentCompatibleType(DataType left, DataType right) { if (left.isNullType()) { return right; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java index 459044100b632d..d9fae844c48912 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUnion.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.LogicalProperties; @@ -28,11 +29,14 @@ import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Union; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.BitSet; @@ -205,6 +209,14 @@ public void computeUniform(DataTrait.Builder builder) { // don't propagate uniform slots } + @Override + public boolean hasUnboundExpression() { + if (!constantExprsList.isEmpty() && children.isEmpty()) { + return false; + } + return super.hasUnboundExpression(); + } + private List mapSlotToIndex(Plan plan, List> equalSlotsList) { Map slotToIndex = new HashMap<>(); for (int i = 0; i < plan.getOutput().size(); i++) { @@ -280,4 +292,73 @@ public void computeEqualSet(DataTrait.Builder builder) { public void computeFd(DataTrait.Builder builder) { // don't generate } + + /** castCommonDataTypeAndNullableByConstants */ + public static Pair>, List> castCommonDataTypeAndNullableByConstants( + List> constantExprsList) { + int columnCount = constantExprsList.isEmpty() ? 0 : constantExprsList.get(0).size(); + Pair, List> commonInfo + = computeCommonDataTypeAndNullable(constantExprsList, columnCount); + List> castedRows = castToCommonType(constantExprsList, commonInfo.key(), columnCount); + List nullables = commonInfo.second; + return Pair.of(castedRows, nullables); + } + + private static Pair, List> computeCommonDataTypeAndNullable( + List> constantExprsList, int columnCount) { + List nullables = Lists.newArrayListWithCapacity(columnCount); + List commonDataTypes = Lists.newArrayListWithCapacity(columnCount); + List firstRow = constantExprsList.get(0); + for (int columnId = 0; columnId < columnCount; columnId++) { + Expression constant = firstRow.get(columnId).child(0); + Pair commonDataTypeAndNullable + = computeCommonDataTypeAndNullable(constant, columnId, constantExprsList); + commonDataTypes.add(commonDataTypeAndNullable.first); + nullables.add(commonDataTypeAndNullable.second); + } + return Pair.of(commonDataTypes, nullables); + } + + private static Pair computeCommonDataTypeAndNullable( + Expression firstRowExpr, int columnId, List> constantExprsList) { + DataType commonDataType = firstRowExpr.getDataType(); + boolean nullable = firstRowExpr.nullable(); + for (int rowId = 1; rowId < constantExprsList.size(); rowId++) { + NamedExpression namedExpression = constantExprsList.get(rowId).get(columnId); + Expression otherConstant = namedExpression.child(0); + nullable |= otherConstant.nullable(); + DataType otherDataType = otherConstant.getDataType(); + commonDataType = getAssignmentCompatibleType(commonDataType, otherDataType); + } + return Pair.of(commonDataType, nullable); + } + + private static List> castToCommonType( + List> rows, List commonDataTypes, int columnCount) { + ImmutableList.Builder> castedConstants + = ImmutableList.builderWithExpectedSize(rows.size()); + for (List row : rows) { + castedConstants.add(castToCommonType(row, commonDataTypes)); + } + return castedConstants.build(); + } + + private static List castToCommonType(List row, List commonTypes) { + ImmutableList.Builder castedRow = ImmutableList.builderWithExpectedSize(row.size()); + boolean changed = false; + for (int columnId = 0; columnId < row.size(); columnId++) { + NamedExpression constantAlias = row.get(columnId); + Expression constant = constantAlias.child(0); + DataType commonType = commonTypes.get(columnId); + if (commonType.equals(constant.getDataType())) { + castedRow.add(constantAlias); + } else { + changed = true; + Expression expression + = TypeCoercionUtils.castIfNotSameTypeStrict(constant, commonType); + castedRow.add((NamedExpression) constantAlias.withChildren(expression)); + } + } + return changed ? castedRow.build() : row; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java index a04912f5119879..65dba188bf02cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java @@ -152,7 +152,7 @@ public int hashCode() { @Override public String toString() { - return Utils.toSqlString("LogicalOlapTableSink[" + id.asInt() + "]", + return Utils.toSqlString("PhysicalOlapTableSink[" + id.asInt() + "]", "outputExprs", outputExprs, "database", database.getFullName(), "targetTable", targetTable.getName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java index b6d3e24ba9963f..6b8a66375c6569 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.visitor; +import org.apache.doris.nereids.analyzer.UnboundInlineTable; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.Command; @@ -99,6 +100,7 @@ public R visitCommand(Command command, C context) { return visit(command, context); } + // ******************************* // relations // ******************************* @@ -130,6 +132,10 @@ public R visitPhysicalSink(PhysicalSink physicalSink, C context) // ******************************* // Logical plans // ******************************* + public R visitUnboundInlineTable(UnboundInlineTable unboundInlineTable, C context) { + return visit(unboundInlineTable, context); + } + public R visitLogicalSqlCache(LogicalSqlCache sqlCache, C context) { return visit(sqlCache, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index f1470d444b91d4..10157496f9b8f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -37,8 +37,8 @@ import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; -import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.plugin.AuditEvent; @@ -173,8 +173,8 @@ private static int countValues(List children) { for (Plan child : children) { if (child instanceof UnboundOneRowRelation) { cnt++; - } else if (child instanceof LogicalInlineTable) { - cnt += ((LogicalInlineTable) child).getConstantExprsList().size(); + } else if (child instanceof InlineTable) { + cnt += ((InlineTable) child).getConstantExprsList().size(); } else if (child instanceof LogicalUnion) { cnt += countValues(child.children()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 15affdff26b0c3..dfa7243e3e249d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -361,6 +361,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_SINGLE_REPLICA_INSERT = "enable_single_replica_insert"; + public static final String ENABLE_FAST_ANALYZE_INSERT_INTO_VALUES = "enable_fast_analyze_into_values"; + public static final String ENABLE_FUNCTION_PUSHDOWN = "enable_function_pushdown"; public static final String ENABLE_EXT_FUNC_PRED_PUSHDOWN = "enable_ext_func_pred_pushdown"; @@ -1510,6 +1512,15 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { needForward = true, varType = VariableAnnotation.EXPERIMENTAL) public boolean enableSingleReplicaInsert = false; + @VariableMgr.VarAttr( + name = ENABLE_FAST_ANALYZE_INSERT_INTO_VALUES, fuzzy = true, + description = { + "跳过大部分的优化规则，快速分析insert into values语句", + "Skip most optimization rules and quickly analyze insert into values statements" + } + ) + private boolean enableFastAnalyzeInsertIntoValues = true; + @VariableMgr.VarAttr(name = ENABLE_FUNCTION_PUSHDOWN, fuzzy = true) public boolean enableFunctionPushdown = false; @@ -3752,8 +3763,6 @@ public boolean isEnableExprTrace() { return enableExprTrace; } - - public boolean isEnableSingleReplicaInsert() { return enableSingleReplicaInsert; } @@ -3762,6 +3771,14 @@ public void setEnableSingleReplicaInsert(boolean enableSingleReplicaInsert) { this.enableSingleReplicaInsert = enableSingleReplicaInsert; } + public boolean isEnableFastAnalyzeInsertIntoValues() { + return enableFastAnalyzeInsertIntoValues; + } + + public void setEnableFastAnalyzeInsertIntoValues(boolean enableFastAnalyzeInsertIntoValues) { + this.enableFastAnalyzeInsertIntoValues = enableFastAnalyzeInsertIntoValues; + } + public boolean isEnableMemtableOnSinkNode() { return enableMemtableOnSinkNode; } diff --git a/regression-test/suites/compression_p0/load.groovy b/regression-test/suites/compression_p0/load.groovy index ff41423094f2e4..06ffb897e4fb1b 100644 --- a/regression-test/suites/compression_p0/load.groovy +++ b/regression-test/suites/compression_p0/load.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_compression", "p0") { +suite("load") { // test snappy compression algorithm def tableName = "test_snappy" diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy index 633ad98d86f556..b448ad406bbde2 100644 --- a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy +++ b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy @@ -32,7 +32,7 @@ suite("test_nestedtypes_insert_into_select", "p0") { test { sql "insert into ast values ('text' , [named_struct('a',1,'b','home'),named_struct('a',2,'b','work')]);" - exception "mismatched input 'named_struct' expecting" + exception "no viable alternative at input '[named_struct'" } @@ -50,6 +50,6 @@ suite("test_nestedtypes_insert_into_select", "p0") { test { sql "insert into ast values ('text' , [named_struct('a',1,'b','home'),named_struct('a',2,'b','work')]);" - exception "mismatched input 'named_struct' expecting" + exception "no viable alternative at input '[named_struct'" } } diff --git a/regression-test/suites/index_p0/load.groovy b/regression-test/suites/index_p0/load.groovy index 174339f148300a..5416a5096329cb 100644 --- a/regression-test/suites/index_p0/load.groovy +++ b/regression-test/suites/index_p0/load.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_bitmap_index_load") { +suite("load") { def tbName = "test_decimal_bitmap_index_multi_page" sql """ diff --git a/regression-test/suites/insert_p0/insert.groovy b/regression-test/suites/insert_p0/insert.groovy index bcab9956c1d4ec..28823815677740 100644 --- a/regression-test/suites/insert_p0/insert.groovy +++ b/regression-test/suites/insert_p0/insert.groovy @@ -112,4 +112,25 @@ suite("insert") { b as (select * from a) select id from a; """ + + sql """ + DROP TABLE IF EXISTS dest; + CREATE TABLE dest ( + l_shipdate DATE NOT NULL, + l_orderkey bigint NOT NULL, + l_linenumber int not null + )ENGINE=OLAP + DUPLICATE KEY(`l_shipdate`, `l_orderkey`) + COMMENT "OLAP" + auto partition by range (date_trunc(`l_shipdate`, 'day')) () + DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + test { + sql("insert into dest values(now(), 0xff, 0xaa)") + exception "Unknown column '0xff' in 'table list' in UNBOUND_OLAP_TABLE_SINK clause" + } } diff --git a/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy b/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy index 9d5abdca1de772..7b1a35eddf03f0 100644 --- a/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy +++ b/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy @@ -240,7 +240,7 @@ suite("insert_group_commit_with_exception") { assertTrue(false) } catch (Exception e) { logger.info("exception : " + e) - assertTrue(e.getMessage().contains("insert into cols should be corresponding to the query output")) + assertTrue(e.getMessage().contains("Column count doesn't match value count")) } } getRowCount(14) diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy index a615c7316bdb56..44fc259a71a1cc 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy @@ -19,7 +19,7 @@ This suite is a two dimensional test case file. It mainly tests the inner join and filter positions. */ -suite("partition_mv_rewrite_dimension_2_2") { +suite("dimension_2_inner_join") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" diff --git a/regression-test/suites/nereids_syntax_p0/explain.groovy b/regression-test/suites/nereids_syntax_p0/explain.groovy index 899bbc2e4e7bd3..5b1bfa1f8ae897 100644 --- a/regression-test/suites/nereids_syntax_p0/explain.groovy +++ b/regression-test/suites/nereids_syntax_p0/explain.groovy @@ -41,7 +41,7 @@ suite("explain") { explain { sql("parsed plan select 100") - contains "UnboundOneRowRelation" + contains "LogicalOneRowRelation" } explain {