From 8d4ef76d4c7587a2e492bf45aac2383bd26ea752 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 23 May 2024 09:39:50 +0800 Subject: [PATCH 1/7] implement prepared statement in nererids --- .../org/apache/doris/nereids/DorisLexer.g4 | 1 + .../org/apache/doris/nereids/DorisParser.g4 | 1 + .../doris/nereids/StatementContext.java | 12 ++ .../nereids/parser/LogicalPlanBuilder.java | 29 ++++ .../rules/analysis/ExpressionAnalyzer.java | 6 + .../trees/expressions/PlaceholderExpr.java | 81 +++++++++ .../trees/expressions/literal/Literal.java | 162 ++++++++++++++++++ .../expressions/literal/TinyIntLiteral.java | 1 + .../visitor/ExpressionVisitor.java | 5 + .../trees/plans/commands/ExecuteCommand.java | 72 ++++++++ .../trees/plans/commands/PreparedCommand.java | 105 ++++++++++++ .../org/apache/doris/qe/ConnectProcessor.java | 3 +- .../doris/qe/MysqlConnectProcessor.java | 136 +++++++++++---- .../apache/doris/qe/PrepareStmtContext.java | 12 +- .../org/apache/doris/qe/StmtExecutor.java | 98 +++++++---- 15 files changed, 648 insertions(+), 76 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index ba6d908d7cacbc..39fae8006647fa 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -392,6 +392,7 @@ ORDER: 'ORDER'; OUTER: 'OUTER'; OUTFILE: 'OUTFILE'; OVER: 'OVER'; +PLACEHOLDER: '?'; OVERWRITE: 'OVERWRITE'; PARAMETER: 'PARAMETER'; PARSED: 'PARSED'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index a57df0dbea7f0f..58791189739b0b 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -880,6 +880,7 @@ constant | LEFT_BRACE (items+=constant COLON items+=constant)? (COMMA items+=constant COLON items+=constant)* RIGHT_BRACE #mapLiteral | LEFT_BRACE items+=constant (COMMA items+=constant)* RIGHT_BRACE #structLiteral + | PLACEHOLDER #placeholder ; comparisonOperator diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 0d658b9e74fa71..5f133d6c6220cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.PlaceholderExpr; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; @@ -144,6 +145,9 @@ public class StatementContext implements Closeable { // table locks private final Stack plannerResources = new Stack<>(); + // placeholder params for prepared statement + private List params; + // for create view support in nereids // key is the start and end position of the sql substring that needs to be replaced, // and value is the new string used for replacement. @@ -490,6 +494,14 @@ public void close() { releasePlannerResources(); } + public List getParams() { + return params; + } + + public void setParams(List params) { + this.params = params; + } + private static class CloseableResource implements Closeable { public final String resourceName; public final String threadName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 79272d0e75a336..9e2e539e23bb52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.parser; +import org.antlr.v4.runtime.CommonToken; import org.apache.doris.analysis.ArithmeticExpr.Operator; import org.apache.doris.analysis.BrokerDesc; import org.apache.doris.analysis.ColumnNullableType; @@ -262,6 +263,7 @@ import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.PlaceholderExpr; import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Regexp; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; @@ -494,6 +496,12 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { private final boolean forCreateView; + private int placeHolderExprId = 0; + + // Sort the parameters with token position to keep the order with original placeholders + // in prepared statement.Otherwise, the order maybe broken + private Map tokenPosToParameters; + public LogicalPlanBuilder() { forCreateView = false; } @@ -1010,6 +1018,11 @@ public List> visitMultiStatements(MultiState } logicalPlans.add(Pair.of( ParserUtils.withOrigin(ctx, () -> (LogicalPlan) visit(statement)), statementContext)); + if (tokenPosToParameters != null) { + List params = new ArrayList<>(tokenPosToParameters.values()); + statementContext.setParams(params); + tokenPosToParameters.clear(); + } } return logicalPlans; } @@ -2322,6 +2335,22 @@ public Literal visitStringLiteral(StringLiteralContext ctx) { return new VarcharLiteral(s, strLength); } + @Override + public Expression visitPlaceholder(DorisParser.PlaceholderContext ctx) { + if (tokenPosToParameters == null) { + tokenPosToParameters = Maps.newTreeMap((pos1, pos2) -> { + int line = pos1.getLine() - pos2.getLine(); + if (line != 0) { + return line; + } + return pos1.getCharPositionInLine() - pos2.getCharPositionInLine(); + }); + } + PlaceholderExpr parameter = new PlaceholderExpr(placeHolderExprId++); + tokenPosToParameters.put(ctx.start, parameter); + return parameter; + } + /** * cast all items to same types. * TODO remove this function after we refactor type coercion. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index 691e4aca1a337d..c662c84bc7e988 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -56,6 +56,7 @@ import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.PlaceholderExpr; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; @@ -541,6 +542,11 @@ public Expression visitNot(Not not, ExpressionRewriteContext context) { return expr; } + @Override + public Expression visitPlaceholderExpr(PlaceholderExpr placeholderExpr, ExpressionRewriteContext context) { + return visit(placeholderExpr.getExpr(), context); + } + @Override public Expression visitComparisonPredicate(ComparisonPredicate cp, ExpressionRewriteContext context) { Expression left = cp.left().accept(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java new file mode 100644 index 00000000000000..c065499e0f92f5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; + +/** + * Placeholder for prepared statement + */ +public class PlaceholderExpr extends Expression { + private final int exprId; + private Expression expr; + + private int mysqlTypeCode; + + public PlaceholderExpr(int exprId) { + this.exprId = exprId; + } + + protected PlaceholderExpr(int exprId, Expression expr) { + this.exprId = exprId; + this.expr = expr; + } + + public Expression getExpr() { + return expr; + } + + public int getExprId() { + return exprId; + } + + public void setExpr(Expression expr) { + this.expr = expr; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitPlaceholderExpr(this, context); + } + + public void setTypeCode(int mysqlTypeCode) { + this.mysqlTypeCode = mysqlTypeCode; + } + + public int getMysqlTypeCode() { + return mysqlTypeCode; + } + + @Override + public boolean nullable() { + return expr.nullable(); + } + + @Override + public String toSql() { + return "?"; + } + + @Override + public DataType getDataType() throws UnboundException { + return expr.getDataType(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index c3369fc28389ac..c770be48cb916d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.catalog.Type; import org.apache.doris.common.Config; +import org.apache.doris.mysql.MysqlProto; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.Expression; @@ -28,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; @@ -36,10 +38,12 @@ import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.IntegralType; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.ByteBuffer; import java.util.Locale; import java.util.Objects; import java.util.Optional; @@ -396,4 +400,162 @@ public boolean isZero() { } return false; } + + /** + ** get paramter length, port from mysql get_param_length + **/ + public static int getParmLen(ByteBuffer data) { + int maxLen = data.remaining(); + if (maxLen < 1) { + return 0; + } + // get and advance 1 byte + int len = MysqlProto.readInt1(data); + if (len == 252) { + if (maxLen < 3) { + return 0; + } + // get and advance 2 bytes + return MysqlProto.readInt2(data); + } else if (len == 253) { + if (maxLen < 4) { + return 0; + } + // get and advance 3 bytes + return MysqlProto.readInt3(data); + } else if (len == 254) { + /* + In our client-server protocol all numbers bigger than 2^24 + stored as 8 bytes with uint8korr. Here we always know that + parameter length is less than 2^4 so we don't look at the second + 4 bytes. But still we need to obey the protocol hence 9 in the + assignment below. + */ + if (maxLen < 9) { + return 0; + } + len = MysqlProto.readInt4(data); + MysqlProto.readFixedString(data, 4); + return len; + } else if (len == 255) { + return 0; + } else { + return len; + } + } + + /** get related Literal from mysql type**/ + public static Literal getLiteralByMysqlType(int mysqlType, ByteBuffer data) throws AnalysisException { + switch (mysqlType) { + // MYSQL_TYPE_TINY + case 1: + return new TinyIntLiteral(data.get()); + // MYSQL_TYPE_SHORT + case 2: + return new SmallIntLiteral((short) data.getChar()); + // MYSQL_TYPE_LONG + case 3: + return new IntegerLiteral(data.getInt()); + // MYSQL_TYPE_LONGLONG + case 8: + return new BigIntLiteral(data.getLong()); + // MYSQL_TYPE_FLOAT + case 4: + return new FloatLiteral(data.getFloat()); + // MYSQL_TYPE_DOUBLE + case 5: + return new DoubleLiteral(data.getDouble()); + // MYSQL_TYPE_DECIMAL + case 0: + // MYSQL_TYPE_NEWDECIMAL + case 246: { + int len = getParmLen(data); + try { + byte[] bytes = new byte[len]; + data.get(bytes); + String value = new String(bytes); + BigDecimal v = new BigDecimal(value); + return new DecimalLiteral(v); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid floating-point literal: {}", e); + } + } + // MYSQL_TYPE_DATE + case 10: { + int len = getParmLen(data); + if (len >= 4) { + int year = (int) data.getChar(); + int month = (int) data.get(); + int day = (int) data.get(); + return new DateLiteral(year, month, day); + } else { + // MIN_DATE + return new DateLiteral(0, 1, 1); + } + } + // MYSQL_TYPE_DATETIME + case 12: + // MYSQL_TYPE_TIMESTAMP + case 7: + // MYSQL_TYPE_TIMESTAMP2 + case 17: { + int len = getParmLen(data); + if (len >= 4) { + int year = (int) data.getChar(); + int month = (int) data.get(); + int day = (int) data.get(); + int hour = 0; + int minute = 0; + int second = 0; + if (len > 4) { + hour = (int) data.get(); + minute = (int) data.get(); + second = (int) data.get(); + } + if (len > 7) { + int microsecond = data.getInt(); + // choose highest scale to keep microsecond value + return new DateTimeLiteral(DateTimeType.INSTANCE, + year, month, day, hour, minute, second, microsecond); + } + return new DateTimeLiteral(DateTimeType.INSTANCE, year, month, day, hour, minute, second); + } else { + return new DateTimeLiteral(0000, 1, 1, 0, 0, 0); + } + } + // MYSQL_TYPE_STRING + case 254: + case 253: { + int strLen = getParmLen(data); + if (strLen > data.remaining()) { + strLen = data.remaining(); + } + byte[] bytes = new byte[strLen]; + data.get(bytes); + String value = new String(bytes); + return new StringLiteral(value); + } + // MYSQL_TYPE_VARCHAR + case 15: { + int strLen = getParmLen(data); + if (strLen > data.remaining()) { + strLen = data.remaining(); + } + byte[] bytes = new byte[strLen]; + data.get(bytes); + String value = new String(bytes); + return new VarcharLiteral(value); + } + default: + throw new AnalysisException("Not support type " + mysqlType); + } + } + + // Parse from binary data, the format follows mysql binary protocal + // see https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_binary_resultset.html. + // Return next offset + public void setupParamFromBinary(ByteBuffer data) { + Preconditions.checkState(false, + "should implement this in derived class. " + toSql()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TinyIntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TinyIntLiteral.java index 08d46c0e5ca69a..feb1ba80c95adf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TinyIntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TinyIntLiteral.java @@ -65,4 +65,5 @@ public int getIntValue() { public Number getNumber() { return value; } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index feea6cfe7d5b2a..b6be307fde4482 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -69,6 +69,7 @@ import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.PlaceholderExpr; import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.Slot; @@ -505,6 +506,10 @@ public R visitMatchPhraseEdge(MatchPhraseEdge matchPhraseEdge, C context) { return visitMatch(matchPhraseEdge, context); } + public R visitPlaceholderExpr(PlaceholderExpr placeholderExpr, C context) { + return null; + } + public R visitAny(Any any, C context) { return visit(any, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java new file mode 100644 index 00000000000000..a0d9e883b1aded --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.RedirectStatus; +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Prepared Statement + */ +public class ExecuteCommand extends LogicalPlanAdapter { + private final String stmtName; + private final List params; + + private StatementContext statementContext; + + public ExecuteCommand(String stmtName, List paramsExpr) { + super(null, null); + this.stmtName = stmtName; + this.params = paramsExpr; + } + + public List getParams() { + return params; + } + + public String getStmtName() { + return stmtName; + } + + @Override + public String toSql() { + return "EXECUTE `" + stmtName + "`" + + params.stream().map(Expression::toSql).collect(Collectors.joining(", ", " USING ", "")); + } + + @Override + public RedirectStatus getRedirectStatus() { + return null; + } + + @Override + public StatementContext getStatementContext() { + if (statementContext == null) { + statementContext = new StatementContext(ConnectContext.get(), new OriginStatement("", 0)); + } + return statementContext; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java new file mode 100644 index 00000000000000..30d9b6a0974d6b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.RedirectStatus; +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.PlaceholderExpr; + +import com.google.common.base.Preconditions; + +import java.util.ArrayList; +import java.util.List; + +/** + * Prepared Statement + */ +public class PreparedCommand extends LogicalPlanAdapter { + protected List params; + private final StatementBase inner; + + private String stmtName; + + public PreparedCommand(String name, StatementBase originStmt, List placeholders) { + super(null, null); + this.inner = originStmt; + this.params = placeholders; + this.stmtName = name; + } + + public String getName() { + return stmtName; + } + + public void setName(String name) { + this.stmtName = name; + } + + public List params() { + return params; + } + + public int getParamLen() { + if (params == null) { + return 0; + } + return params.size(); + } + + public void setParams(List params) { + this.params = params; + } + + public StatementBase getInnerStmt() { + return inner; + } + + public StatementBase assignValues(List values) { + if (params == null) { + return inner; + } + Preconditions.checkArgument(values.size() == params.size(), "Invalid arguments size"); + for (int i = 0; i < params.size(); i++) { + params.get(i).setExpr(values.get(i)); + } + return inner; + } + + public List getLabels() { + List labels = new ArrayList<>(); + if (params == null) { + return labels; + } + for (PlaceholderExpr parameter : params) { + labels.add("$" + parameter.getExprId()); + } + return labels; + } + + @Override + public String toSql() { + return "PREPARE " + stmtName + " FROM " + inner.toSql(); + } + + @Override + public RedirectStatus getRedirectStatus() { + return inner.getRedirectStatus(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 0113d9d1d5ecf9..f6fdab98ef6c0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -257,8 +257,7 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex Exception nereidsSyntaxException = null; long parseSqlStartTime = System.currentTimeMillis(); List cachedStmts = null; - // Nereids do not support prepare and execute now, so forbid prepare command, only process query command - if (mysqlCommand == MysqlCommand.COM_QUERY && sessionVariable.isEnableNereidsPlanner()) { + if (sessionVariable.isEnableNereidsPlanner()) { if (wantToParseSqlFromSqlCache) { cachedStmts = parseFromSqlCache(originStmt); if (cachedStmts != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index d117eeba9493c6..e221eb4d0b986d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -18,15 +18,19 @@ package org.apache.doris.qe; import org.apache.doris.analysis.ExecuteStmt; -import org.apache.doris.analysis.InsertStmt; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.NullLiteral; +import org.apache.doris.analysis.PrepareStmt; import org.apache.doris.analysis.QueryStmt; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.mysql.MysqlProto; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.commands.ExecuteCommand; +import org.apache.doris.nereids.trees.plans.commands.PreparedCommand; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -81,35 +85,12 @@ private void debugPacket() { } } - // process COM_EXECUTE, parse binary row data - // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute.html - private void handleExecute() { - // debugPacket(); - packetBuf = packetBuf.order(ByteOrder.LITTLE_ENDIAN); - // parse stmt_id, flags, params - int stmtId = packetBuf.getInt(); - // flag - packetBuf.get(); - // iteration_count always 1, - packetBuf.getInt(); - if (LOG.isDebugEnabled()) { - LOG.debug("execute prepared statement {}", stmtId); - } - PrepareStmtContext prepareCtx = ctx.getPreparedStmt(String.valueOf(stmtId)); - if (prepareCtx == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("No such statement in context, stmtId:{}", stmtId); - } - ctx.getState().setError(ErrorCode.ERR_UNKNOWN_COM_ERROR, - "msg: Not supported such prepared statement"); - return; - } - ctx.setStartTime(); - if (prepareCtx.stmt.getInnerStmt() instanceof QueryStmt) { + private void handleExecute(PrepareStmt prepareStmt, long stmtId) { + if (prepareStmt.getInnerStmt() instanceof QueryStmt) { ctx.getState().setIsQuery(true); } - prepareCtx.stmt.setIsPrepared(); - int paramCount = prepareCtx.stmt.getParmCount(); + prepareStmt.setIsPrepared(); + int paramCount = prepareStmt.getParmCount(); LOG.debug("execute prepared statement {}, paramCount {}", stmtId, paramCount); // null bitmap String stmtStr = ""; @@ -124,7 +105,7 @@ private void handleExecute() { for (int i = 0; i < paramCount; ++i) { int typeCode = packetBuf.getChar(); LOG.debug("code {}", typeCode); - prepareCtx.stmt.placeholders().get(i).setTypeCode(typeCode); + prepareStmt.placeholders().get(i).setTypeCode(typeCode); } } // parse param data @@ -133,7 +114,7 @@ private void handleExecute() { realValueExprs.add(new NullLiteral()); continue; } - LiteralExpr l = prepareCtx.stmt.placeholders().get(i).createLiteralFromType(); + LiteralExpr l = prepareStmt.placeholders().get(i).createLiteralFromType(); l.setupParamFromBinary(packetBuf); realValueExprs.add(l); } @@ -149,7 +130,7 @@ private void handleExecute() { ctx.setExecutor(executor); executor.execute(); PrepareStmtContext preparedStmtContext = ConnectContext.get().getPreparedStmt(String.valueOf(stmtId)); - if (preparedStmtContext != null && !(preparedStmtContext.stmt.getInnerStmt() instanceof InsertStmt)) { + if (preparedStmtContext != null) { stmtStr = executeStmt.toSql(); } } catch (Throwable e) { @@ -160,7 +141,98 @@ private void handleExecute() { e.getClass().getSimpleName() + ", msg: " + e.getMessage()); } if (!stmtStr.isEmpty()) { - auditAfterExec(stmtStr, prepareCtx.stmt.getInnerStmt(), null, false); + auditAfterExec(stmtStr, prepareStmt.getInnerStmt(), null, false); + } + } + + private void handleExecute(PreparedCommand prepareStmt, long stmtId) { + prepareStmt.setIsPrepared(); + int paramCount = prepareStmt.getParamLen(); + LOG.debug("execute prepared statement {}, paramCount {}", stmtId, paramCount); + // null bitmap + String stmtStr = ""; + try { + List realValueExprs = new ArrayList<>(); + if (paramCount > 0) { + byte[] nullbitmapData = new byte[(paramCount + 7) / 8]; + packetBuf.get(nullbitmapData); + // new_params_bind_flag + if ((int) packetBuf.get() != 0) { + // parse params's types + for (int i = 0; i < paramCount; ++i) { + int typeCode = packetBuf.getChar(); + LOG.debug("code {}", typeCode); + prepareStmt.params().get(i).setTypeCode(typeCode); + } + } + // parse param data + for (int i = 0; i < paramCount; ++i) { + if (isNull(nullbitmapData, i)) { + realValueExprs.add(new org.apache.doris.nereids.trees.expressions.literal.NullLiteral()); + continue; + } + Literal l = Literal.getLiteralByMysqlType( + prepareStmt.params().get(i).getMysqlTypeCode(), packetBuf); + realValueExprs.add(l); + } + } + ExecuteCommand executeStmt = new ExecuteCommand(String.valueOf(stmtId), realValueExprs); + // TODO set real origin statement + executeStmt.setOrigStmt(new OriginStatement("null", 0)); + executeStmt.setUserInfo(ctx.getCurrentUserIdentity()); + if (LOG.isDebugEnabled()) { + LOG.debug("executeStmt {}", executeStmt); + } + executor = new StmtExecutor(ctx, executeStmt); + ctx.setExecutor(executor); + executor.execute(); + stmtStr = executeStmt.toSql(); + } catch (Throwable e) { + // Catch all throwable. + // If reach here, maybe doris bug. + LOG.warn("Process one query failed because unknown reason: ", e); + ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, + e.getClass().getSimpleName() + ", msg: " + e.getMessage()); + } + if (!stmtStr.isEmpty()) { + auditAfterExec(stmtStr, prepareStmt.getInnerStmt(), null, false); + } + } + + // process COM_EXECUTE, parse binary row data + // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute.html + private void handleExecute() { + // debugPacket(); + packetBuf = packetBuf.order(ByteOrder.LITTLE_ENDIAN); + // parse stmt_id, flags, params + int stmtId = packetBuf.getInt(); + // flag + packetBuf.get(); + // iteration_count always 1, + packetBuf.getInt(); + if (LOG.isDebugEnabled()) { + LOG.debug("execute prepared statement {}", stmtId); + } + PrepareStmtContext prepareCtx = ctx.getPreparedStmt(String.valueOf(stmtId)); + if (prepareCtx == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("No such statement in context, stmtId:{}", stmtId); + } + ctx.getState().setError(ErrorCode.ERR_UNKNOWN_COM_ERROR, + "msg: Not supported such prepared statement"); + return; + } + ctx.setStartTime(); + if (prepareCtx.stmt instanceof PrepareStmt) { + // legacy + handleExecute((PrepareStmt) prepareCtx.stmt, stmtId); + } else if (prepareCtx.stmt instanceof PreparedCommand) { + // nererids + handleExecute((PreparedCommand) prepareCtx.stmt, stmtId); + } else { + ctx.getState().setError(ErrorCode.ERR_UNKNOWN_COM_ERROR, + "msg: Not supported such prepared statement"); + return; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PrepareStmtContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/PrepareStmtContext.java index 400f5047b53ac7..3c3707e8b66c3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/PrepareStmtContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PrepareStmtContext.java @@ -18,17 +18,15 @@ package org.apache.doris.qe; import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.PrepareStmt; -import org.apache.doris.planner.OriginalPlanner; +import org.apache.doris.analysis.StatementBase; import org.apache.doris.planner.Planner; -import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; public class PrepareStmtContext { private static final Logger LOG = LogManager.getLogger(PrepareStmtContext.class); - public PrepareStmt stmt; + public StatementBase stmt; public ConnectContext ctx; public Planner planner; public Analyzer analyzer; @@ -37,15 +35,11 @@ public class PrepareStmtContext { // Timestamp in millisecond last command starts at protected volatile long startTime; - public PrepareStmtContext(PrepareStmt stmt, ConnectContext ctx, Planner planner, + public PrepareStmtContext(StatementBase stmt, ConnectContext ctx, Planner planner, Analyzer analyzer, String stmtString) { this.stmt = stmt; this.ctx = ctx; this.planner = planner; - // Only support OriginalPlanner for now - if (planner != null) { - Preconditions.checkState(planner instanceof OriginalPlanner); - } this.analyzer = analyzer; this.stmtString = stmtString; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index db14ff1d8e6533..1ffe4b95d8e70f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -150,8 +150,10 @@ import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromUsingCommand; +import org.apache.doris.nereids.trees.plans.commands.ExecuteCommand; import org.apache.doris.nereids.trees.plans.commands.Forward; import org.apache.doris.nereids.trees.plans.commands.NotAllowFallback; +import org.apache.doris.nereids.trees.plans.commands.PreparedCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; @@ -278,7 +280,7 @@ public class StmtExecutor { private Data.PQueryStatistics.Builder statisticsForAuditLog; private boolean isCached; private String stmtName; - private PrepareStmt prepareStmt = null; + private StatementBase prepareStmt = null; private String mysqlLoadId; // Distinguish from prepare and execute command private boolean isExecuteStmt = false; @@ -690,6 +692,25 @@ private void executeByNereids(TUniqueId queryId) throws Exception { throw new NereidsException(errMsg, new AnalysisException(errMsg)); } } + if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { + if (isForwardToMaster()) { + throw new UserException("Forward master command is not supported for prepare statement"); + } + // register statement and return + handlePrepareStmt(); + return; + } + if (context.getCommand() == MysqlCommand.COM_STMT_EXECUTE) { + ExecuteCommand executeStmt = (ExecuteCommand) parsedStmt; + preparedStmtCtx = context.getPreparedStmt(executeStmt.getStmtName()); + if (null == preparedStmtCtx) { + throw new AnalysisException( + "prepare statement " + executeStmt.getStmtName() + " not found, maybe expired"); + } + PreparedCommand prepareStmt = (PreparedCommand) preparedStmtCtx.stmt; + parsedStmt = prepareStmt.assignValues(executeStmt.getParams()); + logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + } if (logicalPlan instanceof Command) { if (logicalPlan instanceof Forward) { redirectStatus = ((Forward) logicalPlan).toRedirectStatus(); @@ -765,6 +786,7 @@ private void executeByNereids(TUniqueId queryId) throws Exception { LOG.debug("Nereids plan query failed:\n{}", originStmt.originStmt); } LOG.info("NereidsException", e); + e.printStackTrace(); throw new NereidsException(new AnalysisException(e.getMessage(), e)); } profile.getSummaryProfile().setQueryPlanFinishTime(); @@ -1187,8 +1209,8 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException, Interrupt throw new UserException("Could not execute, since `" + execStmt.getName() + "` not exist"); } // parsedStmt may already by set when constructing this StmtExecutor(); - preparedStmtCtx.stmt.asignValues(execStmt.getArgs()); - parsedStmt = preparedStmtCtx.stmt.getInnerStmt(); + ((PrepareStmt) preparedStmtCtx.stmt).asignValues(execStmt.getArgs()); + parsedStmt = ((PrepareStmt) preparedStmtCtx.stmt).getInnerStmt(); planner = preparedStmtCtx.planner; analyzer = preparedStmtCtx.analyzer; prepareStmt = preparedStmtCtx.stmt; @@ -1196,7 +1218,7 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException, Interrupt LOG.debug("already prepared stmt: {}", preparedStmtCtx.stmtString); } isExecuteStmt = true; - if (!preparedStmtCtx.stmt.needReAnalyze()) { + if (!((PrepareStmt) preparedStmtCtx.stmt).needReAnalyze()) { // Return directly to bypass analyze and plan return; } @@ -1218,15 +1240,15 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException, Interrupt if (parsedStmt instanceof PrepareStmt || context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { prepareStmt = new PrepareStmt(parsedStmt, - String.valueOf(context.getEnv().getNextStmtId())); + String.valueOf(String.valueOf(context.getStmtId()))); } else { prepareStmt = (PrepareStmt) parsedStmt; } - prepareStmt.setContext(context); + ((PrepareStmt) prepareStmt).setContext(context); prepareStmt.analyze(analyzer); // Need analyze inner statement - parsedStmt = prepareStmt.getInnerStmt(); - if (prepareStmt.getPreparedType() == PrepareStmt.PreparedType.STATEMENT) { + parsedStmt = ((PrepareStmt) prepareStmt).getInnerStmt(); + if (((PrepareStmt) prepareStmt).getPreparedType() == PrepareStmt.PreparedType.STATEMENT) { // Skip analyze, do it lazy return; } @@ -1344,15 +1366,15 @@ && hasCloudClusterPriv()) { } } if (preparedStmtReanalyzed - && preparedStmtCtx.stmt.getPreparedType() == PrepareStmt.PreparedType.FULL_PREPARED) { - prepareStmt.asignValues(execStmt.getArgs()); + && ((PrepareStmt) preparedStmtCtx.stmt).getPreparedType() == PrepareStmt.PreparedType.FULL_PREPARED) { + ((PrepareStmt) prepareStmt).asignValues(execStmt.getArgs()); if (LOG.isDebugEnabled()) { LOG.debug("update planner and analyzer after prepared statement reanalyzed"); } preparedStmtCtx.planner = planner; preparedStmtCtx.analyzer = analyzer; Preconditions.checkNotNull(preparedStmtCtx.stmt); - preparedStmtCtx.analyzer.setPrepareStmt(preparedStmtCtx.stmt); + preparedStmtCtx.analyzer.setPrepareStmt(((PrepareStmt) preparedStmtCtx.stmt)); } } @@ -1410,9 +1432,9 @@ private void analyzeAndGenerateQueryPlan(TQueryOptions tQueryOptions) throws Use } } if (prepareStmt != null) { - analyzer.setPrepareStmt(prepareStmt); - if (execStmt != null && prepareStmt.getPreparedType() != PreparedType.FULL_PREPARED) { - prepareStmt.asignValues(execStmt.getArgs()); + analyzer.setPrepareStmt(((PrepareStmt) prepareStmt)); + if (execStmt != null && ((PrepareStmt) prepareStmt).getPreparedType() != PreparedType.FULL_PREPARED) { + ((PrepareStmt) prepareStmt).asignValues(execStmt.getArgs()); } } parsedStmt.analyze(analyzer); @@ -1480,9 +1502,10 @@ private void analyzeAndGenerateQueryPlan(TQueryOptions tQueryOptions) throws Use // query re-analyze parsedStmt.reset(); if (prepareStmt != null) { - analyzer.setPrepareStmt(prepareStmt); - if (execStmt != null && prepareStmt.getPreparedType() != PreparedType.FULL_PREPARED) { - prepareStmt.asignValues(execStmt.getArgs()); + analyzer.setPrepareStmt(((PrepareStmt) prepareStmt)); + if (execStmt != null + && ((PrepareStmt) prepareStmt).getPreparedType() != PreparedType.FULL_PREPARED) { + ((PrepareStmt) prepareStmt).asignValues(execStmt.getArgs()); } } analyzer.setReAnalyze(true); @@ -2584,16 +2607,26 @@ private void handleSwitchStmt() throws AnalysisException { } private void handlePrepareStmt() throws Exception { + List labels; + if (prepareStmt == null) { + // must be in nereids, todo remove original planner code + prepareStmt = new PreparedCommand(String.valueOf(context.getStmtId()), + parsedStmt, statementContext.getParams()); + labels = ((PreparedCommand) prepareStmt).getLabels(); + } else { + // legacy planner + labels = ((PrepareStmt) prepareStmt).getColLabelsOfPlaceHolders(); + } // register prepareStmt if (LOG.isDebugEnabled()) { LOG.debug("add prepared statement {}, isBinaryProtocol {}", - prepareStmt.getName(), context.getCommand() == MysqlCommand.COM_STMT_PREPARE); + prepareStmt.toSql(), context.getCommand() == MysqlCommand.COM_STMT_PREPARE); } - context.addPreparedStmt(prepareStmt.getName(), + context.addPreparedStmt(String.valueOf(context.getStmtId()), new PrepareStmtContext(prepareStmt, - context, planner, analyzer, prepareStmt.getName())); + context, planner, analyzer, String.valueOf(context.getStmtId()))); if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { - sendStmtPrepareOK(); + sendStmtPrepareOK((int) context.getStmtId(), labels); } } @@ -2697,19 +2730,19 @@ private List exprToStringType(List exprs) { return exprs.stream().map(e -> PrimitiveType.STRING).collect(Collectors.toList()); } - private void sendStmtPrepareOK() throws IOException { + private void sendStmtPrepareOK(int stmtId, List labels) throws IOException { Preconditions.checkState(context.getConnectType() == ConnectType.MYSQL); // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_prepare.html#sect_protocol_com_stmt_prepare_response serializer.reset(); // 0x00 OK serializer.writeInt1(0); // statement_id - serializer.writeInt4(Integer.valueOf(prepareStmt.getName())); + serializer.writeInt4(stmtId); // num_columns int numColumns = 0; serializer.writeInt2(numColumns); // num_params - int numParams = prepareStmt.getColLabelsOfPlaceHolders().size(); + int numParams = labels.size(); serializer.writeInt2(numParams); // reserved_1 serializer.writeInt1(0); @@ -2718,14 +2751,12 @@ private void sendStmtPrepareOK() throws IOException { // send field one by one // TODO use real type instead of string, for JDBC client it's ok // but for other client, type should be correct - List types = exprToStringType(prepareStmt.getPlaceHolderExprList()); - List colNames = prepareStmt.getColLabelsOfPlaceHolders(); - if (LOG.isDebugEnabled()) { - LOG.debug("sendFields {}, {}", colNames, types); - } + // List types = exprToStringType(labels); + List colNames = labels; for (int i = 0; i < colNames.size(); ++i) { serializer.reset(); - serializer.writeField(colNames.get(i), Type.fromPrimitiveType(types.get(i))); + // serializer.writeField(colNames.get(i), Type.fromPrimitiveType(types.get(i))); + serializer.writeField(colNames.get(i), Type.STRING); context.getMysqlChannel().sendOnePacket(serializer.toByteBuffer()); } serializer.reset(); @@ -2754,14 +2785,15 @@ private void sendFields(List colNames, List types) throws IOExcept // send field one by one for (int i = 0; i < colNames.size(); ++i) { serializer.reset(); - if (prepareStmt != null && isExecuteStmt) { + if (prepareStmt != null && prepareStmt instanceof PrepareStmt + && context.getCommand() == MysqlCommand.COM_STMT_EXECUTE) { // Using PreparedStatment pre serializedField to avoid serialize each time // we send a field - byte[] serializedField = prepareStmt.getSerializedField(colNames.get(i)); + byte[] serializedField = ((PrepareStmt) prepareStmt).getSerializedField(colNames.get(i)); if (serializedField == null) { serializer.writeField(colNames.get(i), types.get(i)); serializedField = serializer.toArray(); - prepareStmt.setSerializedField(colNames.get(i), serializedField); + ((PrepareStmt) prepareStmt).setSerializedField(colNames.get(i), serializedField); } context.getMysqlChannel().sendOnePacket(ByteBuffer.wrap(serializedField)); } else { From 463c75d663a73990f618fc8161805613a7340ffb Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 23 May 2024 21:34:13 +0800 Subject: [PATCH 2/7] fix profile and auditlog --- .../java/org/apache/doris/qe/MysqlConnectProcessor.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index e221eb4d0b986d..c3c127e389a697 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -140,9 +140,7 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) { ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getClass().getSimpleName() + ", msg: " + e.getMessage()); } - if (!stmtStr.isEmpty()) { - auditAfterExec(stmtStr, prepareStmt.getInnerStmt(), null, false); - } + auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); } private void handleExecute(PreparedCommand prepareStmt, long stmtId) { @@ -183,6 +181,7 @@ private void handleExecute(PreparedCommand prepareStmt, long stmtId) { if (LOG.isDebugEnabled()) { LOG.debug("executeStmt {}", executeStmt); } + executeStmt.setOrigStmt(prepareStmt.getInnerStmt().getOrigStmt()); executor = new StmtExecutor(ctx, executeStmt); ctx.setExecutor(executor); executor.execute(); @@ -194,9 +193,7 @@ private void handleExecute(PreparedCommand prepareStmt, long stmtId) { ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getClass().getSimpleName() + ", msg: " + e.getMessage()); } - if (!stmtStr.isEmpty()) { - auditAfterExec(stmtStr, prepareStmt.getInnerStmt(), null, false); - } + auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); } // process COM_EXECUTE, parse binary row data From ea17de36538c43149f4d66d40fd69b5fa14d40da Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 23 May 2024 22:24:12 +0800 Subject: [PATCH 3/7] refactor some code --- .../apache/doris/catalog/MysqlColType.java | 15 ++ .../nereids/parser/LogicalPlanBuilder.java | 1 - .../trees/expressions/PlaceholderExpr.java | 7 +- .../trees/expressions/literal/Literal.java | 195 +++++++++--------- .../trees/plans/commands/ExecuteCommand.java | 3 +- .../trees/plans/commands/PreparedCommand.java | 6 + .../org/apache/doris/qe/StmtExecutor.java | 1 - 7 files changed, 119 insertions(+), 109 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java index 0008f16ab10101..cd48b0afd94611 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java @@ -17,6 +17,9 @@ package org.apache.doris.catalog; +import java.util.HashMap; +import java.util.Map; + // MySQL column type // TYPE codes are defined in the file 'mysql/include/mysql_com.h' enum enum_field_types // which is also demostrated in @@ -55,6 +58,14 @@ public enum MysqlColType { MYSQL_TYPE_GEOMETRY(255, "GEOMETRY", "GEOMETRY"), MYSQL_TYPE_MAP(400, "MAP", "MAP"); + private static final Map CODE_MAP = new HashMap<>(); + + static { + for (MysqlColType type : MysqlColType.values()) { + CODE_MAP.put(type.code, type); + } + } + private MysqlColType(int code, String desc, String jdbcColumnTypeName) { this.code = code; this.desc = desc; @@ -77,6 +88,10 @@ public int getCode() { return code; } + public static MysqlColType fromCode(int code) { + return CODE_MAP.get(code); + } + public String getJdbcColumnTypeName() { return jdbcColumnTypeName; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 9e2e539e23bb52..3d0fa15c4a38e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.parser; -import org.antlr.v4.runtime.CommonToken; import org.apache.doris.analysis.ArithmeticExpr.Operator; import org.apache.doris.analysis.BrokerDesc; import org.apache.doris.analysis.ColumnNullableType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java index c065499e0f92f5..e0dd67357b147b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.expressions; +import org.apache.doris.catalog.MysqlColType; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; @@ -28,7 +29,7 @@ public class PlaceholderExpr extends Expression { private final int exprId; private Expression expr; - private int mysqlTypeCode; + private MysqlColType mysqlTypeCode; public PlaceholderExpr(int exprId) { this.exprId = exprId; @@ -57,10 +58,10 @@ public R accept(ExpressionVisitor visitor, C context) { } public void setTypeCode(int mysqlTypeCode) { - this.mysqlTypeCode = mysqlTypeCode; + this.mysqlTypeCode = MysqlColType.fromCode(mysqlTypeCode); } - public int getMysqlTypeCode() { + public MysqlColType getMysqlTypeCode() { return mysqlTypeCode; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index c770be48cb916d..8ae54b4418a2bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.BoolLiteral; import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.catalog.MysqlColType; import org.apache.doris.catalog.Type; import org.apache.doris.common.Config; import org.apache.doris.mysql.MysqlProto; @@ -38,7 +39,6 @@ import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.IntegralType; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import java.math.BigDecimal; @@ -444,118 +444,109 @@ public static int getParmLen(ByteBuffer data) { } } - /** get related Literal from mysql type**/ - public static Literal getLiteralByMysqlType(int mysqlType, ByteBuffer data) throws AnalysisException { + /** + * Retrieves a Literal object based on the MySQL type and the data provided. + * + * @param mysqlType the MySQL type identifier + * @param data the ByteBuffer containing the data + * @return a Literal object corresponding to the MySQL type + * @throws AnalysisException if the MySQL type is unsupported or if data conversion fails + */ + public static Literal getLiteralByMysqlType(MysqlColType mysqlType, ByteBuffer data) throws AnalysisException { switch (mysqlType) { - // MYSQL_TYPE_TINY - case 1: + case MYSQL_TYPE_TINY: return new TinyIntLiteral(data.get()); - // MYSQL_TYPE_SHORT - case 2: + case MYSQL_TYPE_SHORT: return new SmallIntLiteral((short) data.getChar()); - // MYSQL_TYPE_LONG - case 3: + case MYSQL_TYPE_LONG: return new IntegerLiteral(data.getInt()); - // MYSQL_TYPE_LONGLONG - case 8: + case MYSQL_TYPE_LONGLONG: return new BigIntLiteral(data.getLong()); - // MYSQL_TYPE_FLOAT - case 4: + case MYSQL_TYPE_FLOAT: return new FloatLiteral(data.getFloat()); - // MYSQL_TYPE_DOUBLE - case 5: + case MYSQL_TYPE_DOUBLE: return new DoubleLiteral(data.getDouble()); - // MYSQL_TYPE_DECIMAL - case 0: - // MYSQL_TYPE_NEWDECIMAL - case 246: { - int len = getParmLen(data); - try { - byte[] bytes = new byte[len]; - data.get(bytes); - String value = new String(bytes); - BigDecimal v = new BigDecimal(value); - return new DecimalLiteral(v); - } catch (NumberFormatException e) { - throw new AnalysisException("Invalid floating-point literal: {}", e); - } - } - // MYSQL_TYPE_DATE - case 10: { - int len = getParmLen(data); - if (len >= 4) { - int year = (int) data.getChar(); - int month = (int) data.get(); - int day = (int) data.get(); - return new DateLiteral(year, month, day); - } else { - // MIN_DATE - return new DateLiteral(0, 1, 1); - } - } - // MYSQL_TYPE_DATETIME - case 12: - // MYSQL_TYPE_TIMESTAMP - case 7: - // MYSQL_TYPE_TIMESTAMP2 - case 17: { - int len = getParmLen(data); - if (len >= 4) { - int year = (int) data.getChar(); - int month = (int) data.get(); - int day = (int) data.get(); - int hour = 0; - int minute = 0; - int second = 0; - if (len > 4) { - hour = (int) data.get(); - minute = (int) data.get(); - second = (int) data.get(); - } - if (len > 7) { - int microsecond = data.getInt(); - // choose highest scale to keep microsecond value - return new DateTimeLiteral(DateTimeType.INSTANCE, - year, month, day, hour, minute, second, microsecond); - } - return new DateTimeLiteral(DateTimeType.INSTANCE, year, month, day, hour, minute, second); - } else { - return new DateTimeLiteral(0000, 1, 1, 0, 0, 0); - } - } - // MYSQL_TYPE_STRING - case 254: - case 253: { - int strLen = getParmLen(data); - if (strLen > data.remaining()) { - strLen = data.remaining(); - } - byte[] bytes = new byte[strLen]; - data.get(bytes); - String value = new String(bytes); - return new StringLiteral(value); + case MYSQL_TYPE_DECIMAL: + case MYSQL_TYPE_NEWDECIMAL: + return handleDecimalLiteral(data); + case MYSQL_TYPE_DATE: + return handleDateLiteral(data); + case MYSQL_TYPE_DATETIME: + case MYSQL_TYPE_TIMESTAMP: + case MYSQL_TYPE_TIMESTAMP2: + return handleDateTimeLiteral(data); + case MYSQL_TYPE_STRING: + case MYSQL_TYPE_VARSTRING: + return handleStringLiteral(data); + case MYSQL_TYPE_VARCHAR: + return handleVarcharLiteral(data); + default: + throw new AnalysisException("Unsupported MySQL type: " + mysqlType); + } + } + + private static DecimalLiteral handleDecimalLiteral(ByteBuffer data) throws AnalysisException { + int len = getParmLen(data); + byte[] bytes = new byte[len]; + data.get(bytes); + try { + String value = new String(bytes); + BigDecimal v = new BigDecimal(value); + return new DecimalLiteral(v); + } catch (NumberFormatException e) { + throw new AnalysisException("Invalid decimal literal", e); + } + } + + private static DateLiteral handleDateLiteral(ByteBuffer data) { + int len = getParmLen(data); + if (len >= 4) { + int year = (int) data.getChar(); + int month = (int) data.get(); + int day = (int) data.get(); + return new DateLiteral(year, month, day); + } else { + return new DateLiteral(0, 1, 1); + } + } + + private static DateTimeLiteral handleDateTimeLiteral(ByteBuffer data) { + int len = getParmLen(data); + if (len >= 4) { + int year = (int) data.getChar(); + int month = (int) data.get(); + int day = (int) data.get(); + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; + if (len > 4) { + hour = (int) data.get(); + minute = (int) data.get(); + second = (int) data.get(); } - // MYSQL_TYPE_VARCHAR - case 15: { - int strLen = getParmLen(data); - if (strLen > data.remaining()) { - strLen = data.remaining(); - } - byte[] bytes = new byte[strLen]; - data.get(bytes); - String value = new String(bytes); - return new VarcharLiteral(value); + if (len > 7) { + microsecond = data.getInt(); } - default: - throw new AnalysisException("Not support type " + mysqlType); + return new DateTimeLiteral(DateTimeType.INSTANCE, year, month, day, hour, minute, second, microsecond); + } else { + return new DateTimeLiteral(0, 1, 1, 0, 0, 0); } } - // Parse from binary data, the format follows mysql binary protocal - // see https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_binary_resultset.html. - // Return next offset - public void setupParamFromBinary(ByteBuffer data) { - Preconditions.checkState(false, - "should implement this in derived class. " + toSql()); + private static StringLiteral handleStringLiteral(ByteBuffer data) { + int strLen = getParmLen(data); + strLen = Math.min(strLen, data.remaining()); + byte[] bytes = new byte[strLen]; + data.get(bytes); + return new StringLiteral(new String(bytes)); + } + + private static VarcharLiteral handleVarcharLiteral(ByteBuffer data) { + int strLen = getParmLen(data); + strLen = Math.min(strLen, data.remaining()); + byte[] bytes = new byte[strLen]; + data.get(bytes); + return new VarcharLiteral(new String(bytes)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java index a0d9e883b1aded..209a0bc288f911 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.analysis.RedirectStatus; -import org.apache.doris.analysis.StatementBase; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.expressions.Expression; @@ -31,7 +30,7 @@ /** * Prepared Statement */ -public class ExecuteCommand extends LogicalPlanAdapter { +public class ExecuteCommand extends LogicalPlanAdapter { private final String stmtName; private final List params; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java index 30d9b6a0974d6b..19abb104fb2609 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java @@ -71,6 +71,9 @@ public StatementBase getInnerStmt() { return inner; } + /** + * assign real value to placeholders and return the assigned statement + */ public StatementBase assignValues(List values) { if (params == null) { return inner; @@ -82,6 +85,9 @@ public StatementBase assignValues(List values) { return inner; } + /** + * return the labels of paramters + */ public List getLabels() { List labels = new ArrayList<>(); if (params == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 1ffe4b95d8e70f..68b4116827cece 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -786,7 +786,6 @@ private void executeByNereids(TUniqueId queryId) throws Exception { LOG.debug("Nereids plan query failed:\n{}", originStmt.originStmt); } LOG.info("NereidsException", e); - e.printStackTrace(); throw new NereidsException(new AnalysisException(e.getMessage(), e)); } profile.getSummaryProfile().setQueryPlanFinishTime(); From 8910771640ba60ffc16c17d22093ad1c09076cc5 Mon Sep 17 00:00:00 2001 From: eldenmoon <15605149486@163.com> Date: Thu, 23 May 2024 22:28:06 +0800 Subject: [PATCH 4/7] add test case --- .../data/prepared_stmt_p0/prepared_stmt.out | 53 ++-- .../point_query_p0/test_point_query.groovy | 5 +- .../prepared_stmt_p0/prepared_stmt.groovy | 249 ++++++++---------- 3 files changed, 140 insertions(+), 167 deletions(-) diff --git a/regression-test/data/prepared_stmt_p0/prepared_stmt.out b/regression-test/data/prepared_stmt_p0/prepared_stmt.out index 396ee931683c9e..bd01fbf2c59582 100644 --- a/regression-test/data/prepared_stmt_p0/prepared_stmt.out +++ b/regression-test/data/prepared_stmt_p0/prepared_stmt.out @@ -1,30 +1,30 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql -- -1231 119291.110000000 ddd laooq \N 2020-01-01 12:36:38 \N 1022-01-01 ["2022-01-01 11:30:38", "2022-01-01 11:30:38", "2022-01-01 11:30:38"] -1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01 12:36:38 522.762 2022-01-01 ["2023-01-01 11:30:38", "2023-01-01 11:30:38"] -1233 1.392932911 yyy laooq 2024-01-02 2020-01-01 12:36:38 52.862 3022-01-01 ["2024-01-01 11:30:38", "2024-01-01 11:30:38", "2024-01-01 11:30:38"] -1234 12919291.129191137 xxddd laooq 2025-01-02 2020-01-01 12:36:38 552.872 4022-01-01 ["2025-01-01 11:30:38", "2025-01-01 11:30:38", "2025-01-01 11:30:38"] -1235 991129292901.111380000 dd \N 2120-01-02 2020-01-01 12:36:38 652.692 5022-01-01 [] -1236 100320.111390000 laa ddd laooq 2220-01-02 2020-01-01 12:36:38 2.7692 6022-01-01 [null] -1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 ["2022-01-01 11:30:38", "2022-01-01 11:30:38", "2022-01-01 11:30:38"] +1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01T12:36:38 522.762 2022-01-01 ["2023-01-01 11:30:38", "2023-01-01 11:30:38"] +1233 1.392932911 yyy laooq 2024-01-02 2020-01-01T12:36:38 52.862 3022-01-01 ["2024-01-01 11:30:38", "2024-01-01 11:30:38", "2024-01-01 11:30:38"] +1234 12919291.129191137 xxddd laooq 2025-01-02 2020-01-01T12:36:38 552.872 4022-01-01 ["2025-01-01 11:30:38", "2025-01-01 11:30:38", "2025-01-01 11:30:38"] +1235 991129292901.111380000 dd \N 2120-01-02 2020-01-01T12:36:38 652.692 5022-01-01 [] +1236 100320.111390000 laa ddd laooq 2220-01-02 2020-01-01T12:36:38 2.7692 6022-01-01 [null] +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] -- !sql -- -1231 119291.110000000 ddd laooq \N 2020-01-01 12:36:38 \N 1022-01-01 ["2022-01-01 11:30:38", "2022-01-01 11:30:38", "2022-01-01 11:30:38"] -1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01 12:36:38 522.762 2022-01-01 ["2023-01-01 11:30:38", "2023-01-01 11:30:38"] -1233 1.392932911 yyy laooq 2024-01-02 2020-01-01 12:36:38 52.862 3022-01-01 ["2024-01-01 11:30:38", "2024-01-01 11:30:38", "2024-01-01 11:30:38"] -1234 12919291.129191137 xxddd laooq 2025-01-02 2020-01-01 12:36:38 552.872 4022-01-01 ["2025-01-01 11:30:38", "2025-01-01 11:30:38", "2025-01-01 11:30:38"] -1235 991129292901.111380000 dd \N 2120-01-02 2020-01-01 12:36:38 652.692 5022-01-01 [] -1236 100320.111390000 laa ddd laooq 2220-01-02 2020-01-01 12:36:38 2.7692 6022-01-01 [null] -1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 ["2022-01-01 11:30:38", "2022-01-01 11:30:38", "2022-01-01 11:30:38"] +1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01T12:36:38 522.762 2022-01-01 ["2023-01-01 11:30:38", "2023-01-01 11:30:38"] +1233 1.392932911 yyy laooq 2024-01-02 2020-01-01T12:36:38 52.862 3022-01-01 ["2024-01-01 11:30:38", "2024-01-01 11:30:38", "2024-01-01 11:30:38"] +1234 12919291.129191137 xxddd laooq 2025-01-02 2020-01-01T12:36:38 552.872 4022-01-01 ["2025-01-01 11:30:38", "2025-01-01 11:30:38", "2025-01-01 11:30:38"] +1235 991129292901.111380000 dd \N 2120-01-02 2020-01-01T12:36:38 652.692 5022-01-01 [] +1236 100320.111390000 laa ddd laooq 2220-01-02 2020-01-01T12:36:38 2.7692 6022-01-01 [null] +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] -- !select0 -- -1231 119291.110000000 ddd laooq \N 2020-01-01 12:36:38 \N 1022-01-01 ["2022-01-01 11:30:38", "2022-01-01 11:30:38", "2022-01-01 11:30:38"] +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 ["2022-01-01 11:30:38", "2022-01-01 11:30:38", "2022-01-01 11:30:38"] -- !select0 -- -1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01 12:36:38 522.762 2022-01-01 ["2023-01-01 11:30:38", "2023-01-01 11:30:38"] +1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01T12:36:38 522.762 2022-01-01 ["2023-01-01 11:30:38", "2023-01-01 11:30:38"] -- !select0 -- -1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01 12:36:38 522.762 2022-01-01 ["2023-01-01 11:30:38", "2023-01-01 11:30:38"] +1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01T12:36:38 522.762 2022-01-01 ["2023-01-01 11:30:38", "2023-01-01 11:30:38"] -- !select1 -- 646464 xxxx--- @@ -36,13 +36,13 @@ 787878 yyyy--- -- !select2 -- -1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] 1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] 1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] -- !select2 -- -1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] 1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] 1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] -- !select2 -- -1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] 1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] 1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] -- !select3 -- 1 1 user1 30 1234 12345 @@ -53,3 +53,16 @@ -- !select5 -- 1 +-- !select6 -- +2 1 user1 \N 1234.1111 xxxlalala + +-- !select7 -- +2 1 user1 \N 1111111 1111111 + +-- !select8 -- +1 +1 + +-- !select9 -- +2 + diff --git a/regression-test/suites/point_query_p0/test_point_query.groovy b/regression-test/suites/point_query_p0/test_point_query.groovy index 8d1b62108bc2c1..7bea03fc9f5eed 100644 --- a/regression-test/suites/point_query_p0/test_point_query.groovy +++ b/regression-test/suites/point_query_p0/test_point_query.groovy @@ -17,7 +17,7 @@ import java.math.BigDecimal; -suite("test_point_query") { +suite("test_point_query", "nonConcurrent") { def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); @@ -30,7 +30,7 @@ suite("test_point_query") { try { set_be_config.call("disable_storage_row_cache", "false") // nereids do not support point query now - sql """set enable_nereids_planner=false""" + sql """set global enable_nereids_planner=false""" def user = context.config.jdbcUser def password = context.config.jdbcPassword @@ -274,5 +274,6 @@ suite("test_point_query") { sql "select /*+ SET_VAR(enable_nereids_planner=false) */ substr(RPTNO,2,5) from test_ODS_EBA_LLREPORT where RPTNO = '567890'" } finally { set_be_config.call("disable_storage_row_cache", "true") + sql """set global enable_nereids_planner=true""" } } \ No newline at end of file diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy index 46f8d7629caf10..f4c13fd04b06f3 100644 --- a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy +++ b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy @@ -22,158 +22,117 @@ suite("test_prepared_stmt", "nonConcurrent") { def user = context.config.jdbcUser def password = context.config.jdbcPassword def url = context.config.jdbcUrl + "&useServerPrepStmts=true" - // def url = context.config.jdbcUrl - def result1 = connect(user=user, password=password, url=url) { sql "set global enable_server_side_prepared_statement = true" - def insert_prepared = { stmt, k1 , k2, k3, k4, k5, k6, k7, k8, k9 -> - java.text.SimpleDateFormat formater = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - if (k1 == null) { - stmt.setNull(1, java.sql.Types.INTEGER); - } else { - stmt.setInt(1, k1) - } - if (k2 == null) { - stmt.setNull(2, java.sql.Types.DECIMAL); - } else { - stmt.setBigDecimal(2, k2) - } - if (k3 == null) { - stmt.setNull(3, java.sql.Types.VARCHAR); - } else { - stmt.setString(3, k3) - } - if (k4 == null) { - stmt.setNull(4, java.sql.Types.VARCHAR); - } else { - stmt.setString(4, k4) - } - if (k5 == null) { - stmt.setNull(5, java.sql.Types.DATE); - } else { - stmt.setDate(5, java.sql.Date.valueOf(k5)) - } - if (k6 == null) { - stmt.setNull(6, java.sql.Types.TIMESTAMP); - } else { - stmt.setTimestamp(6, new java.sql.Timestamp(formater.parse(k6).getTime())) - } - if (k7 == null) { - stmt.setNull(7, java.sql.Types.FLOAT); - } else { - stmt.setFloat(7, k7) - } - if (k8 == null) { - stmt.setNull(8, java.sql.Types.DATE); - } else { - stmt.setTimestamp(8, new java.sql.Timestamp(formater.parse(k8).getTime())) - } - if (k9 == null) { - stmt.setNull(9, java.sql.Types.VARCHAR); - } else { - stmt.setString(9, k9) - } - exec stmt - } - - sql """DROP TABLE IF EXISTS ${tableName} """ - sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - `k1` int(11) NULL COMMENT "", - `k2` decimalv3(27, 9) NULL COMMENT "", - `k3` varchar(30) NULL COMMENT "", - `k4` varchar(30) NULL COMMENT "", - `k5` date NULL COMMENT "", - `k6` datetime NULL COMMENT "", - `k7` float NULL COMMENT "", - `k8` datev2 NULL COMMENT "", - `k9` array NULL COMMENT "" - ) ENGINE=OLAP - DUPLICATE KEY(`k1`, `k2`, `k3`) - DISTRIBUTED BY HASH(`k1`, k2, k3) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "light_schema_change" = "true", - "storage_format" = "V2" - ) - """ - - // def insert_stmt = prepareStatement """ INSERT INTO ${tableName} VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?) """ - // assertEquals(insert_stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement); - // insert_prepared insert_stmt, 1231, 119291.11, "ddd", "laooq", null, "2020-01-01 12:36:38", null, "1022-01-01 11:30:38", "[2022-01-01 11:30:38, 2022-01-01 11:30:38, 2022-01-01 11:30:38]" - // insert_prepared insert_stmt, 1232, 12222.99121135, "xxx", "laooq", "2023-01-02", "2020-01-01 12:36:38", 522.762, "2022-01-01 11:30:38", "[2023-01-01 11:30:38, 2023-01-01 11:30:38]" - // insert_prepared insert_stmt, 1233, 1.392932911136, "yyy", "laooq", "2024-01-02", "2020-01-01 12:36:38", 52.862, "3022-01-01 11:30:38", "[2024-01-01 11:30:38, 2024-01-01 11:30:38, 2024-01-01 11:30:38]" - // insert_prepared insert_stmt, 1234, 12919291.129191137, "xxddd", "laooq", "2025-01-02", "2020-01-01 12:36:38", 552.872, "4022-01-01 11:30:38", "[2025-01-01 11:30:38, 2025-01-01 11:30:38, 2025-01-01 11:30:38]" - // insert_prepared insert_stmt, 1235, 991129292901.11138, "dd", null, "2120-01-02", "2020-01-01 12:36:38", 652.692, "5022-01-01 11:30:38", "[]" - // insert_prepared insert_stmt, 1236, 100320.11139, "laa ddd", "laooq", "2220-01-02", "2020-01-01 12:36:38", 2.7692, "6022-01-01 11:30:38", "[null]" - // insert_prepared insert_stmt, 1237, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", "[2025-01-01 11:30:38]" + def result1 = connect(user=user, password=password, url=url) { + sql """DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL COMMENT "", + `k2` decimalv3(27, 9) NULL COMMENT "", + `k3` varchar(30) NULL COMMENT "", + `k4` varchar(30) NULL COMMENT "", + `k5` date NULL COMMENT "", + `k6` datetime NULL COMMENT "", + `k7` float NULL COMMENT "", + `k8` datev2 NULL COMMENT "", + `k9` array NULL COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`, `k3`) + DISTRIBUTED BY HASH(`k1`, k2, k3) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "light_schema_change" = "true", + "storage_format" = "V2" + ) + """ - sql """ INSERT INTO ${tableName} VALUES(1231, 119291.11, "ddd", "laooq", null, "2020-01-01 12:36:38", null, "1022-01-01 11:30:38", "[2022-01-01 11:30:38, 2022-01-01 11:30:38, 2022-01-01 11:30:38]") """ - sql """ INSERT INTO ${tableName} VALUES(1232, 12222.99121135, "xxx", "laooq", "2023-01-02", "2020-01-01 12:36:38", 522.762, "2022-01-01 11:30:38", "[2023-01-01 11:30:38, 2023-01-01 11:30:38]") """ - sql """ INSERT INTO ${tableName} VALUES(1233, 1.392932911136, "yyy", "laooq", "2024-01-02", "2020-01-01 12:36:38", 52.862, "3022-01-01 11:30:38", "[2024-01-01 11:30:38, 2024-01-01 11:30:38, 2024-01-01 11:30:38]") """ - sql """ INSERT INTO ${tableName} VALUES(1234, 12919291.129191137, "xxddd", "laooq", "2025-01-02", "2020-01-01 12:36:38", 552.872, "4022-01-01 11:30:38", "[2025-01-01 11:30:38, 2025-01-01 11:30:38, 2025-01-01 11:30:38]") """ - sql """ INSERT INTO ${tableName} VALUES(1235, 991129292901.11138, "dd", null, "2120-01-02", "2020-01-01 12:36:38", 652.692, "5022-01-01 11:30:38", "[]") """ - sql """ INSERT INTO ${tableName} VALUES(1236, 100320.11139, "laa ddd", "laooq", "2220-01-02", "2020-01-01 12:36:38", 2.7692, "6022-01-01 11:30:38", "[null]") """ - sql """ INSERT INTO ${tableName} VALUES(1237, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", "[2025-01-01 11:30:38]") """ - sql """sync""" + sql """ INSERT INTO ${tableName} VALUES(1231, 119291.11, "ddd", "laooq", null, "2020-01-01 12:36:38", null, "1022-01-01 11:30:38", "[2022-01-01 11:30:38, 2022-01-01 11:30:38, 2022-01-01 11:30:38]") """ + sql """ INSERT INTO ${tableName} VALUES(1232, 12222.99121135, "xxx", "laooq", "2023-01-02", "2020-01-01 12:36:38", 522.762, "2022-01-01 11:30:38", "[2023-01-01 11:30:38, 2023-01-01 11:30:38]") """ + sql """ INSERT INTO ${tableName} VALUES(1233, 1.392932911136, "yyy", "laooq", "2024-01-02", "2020-01-01 12:36:38", 52.862, "3022-01-01 11:30:38", "[2024-01-01 11:30:38, 2024-01-01 11:30:38, 2024-01-01 11:30:38]") """ + sql """ INSERT INTO ${tableName} VALUES(1234, 12919291.129191137, "xxddd", "laooq", "2025-01-02", "2020-01-01 12:36:38", 552.872, "4022-01-01 11:30:38", "[2025-01-01 11:30:38, 2025-01-01 11:30:38, 2025-01-01 11:30:38]") """ + sql """ INSERT INTO ${tableName} VALUES(1235, 991129292901.11138, "dd", null, "2120-01-02", "2020-01-01 12:36:38", 652.692, "5022-01-01 11:30:38", "[]") """ + sql """ INSERT INTO ${tableName} VALUES(1236, 100320.11139, "laa ddd", "laooq", "2220-01-02", "2020-01-01 12:36:38", 2.7692, "6022-01-01 11:30:38", "[null]") """ + sql """ INSERT INTO ${tableName} VALUES(1237, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", "[2025-01-01 11:30:38]") """ + sql """sync""" - qt_sql """select * from ${tableName} order by 1, 2, 3""" - qt_sql """select * from ${tableName} order by 1, 2, 3""" + qt_sql """select * from ${tableName} order by 1, 2, 3""" + qt_sql """select * from ${tableName} order by 1, 2, 3""" - def stmt_read = prepareStatement "select * from ${tableName} where k1 = ? order by k1" - assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); - stmt_read.setInt(1, 1231) - qe_select0 stmt_read - stmt_read.setInt(1, 1232) - qe_select0 stmt_read - qe_select0 stmt_read - def stmt_read1 = prepareStatement "select hex(k3), ? from ${tableName} where k1 = ? order by 1" - assertEquals(stmt_read1.class, com.mysql.cj.jdbc.ServerPreparedStatement); - stmt_read1.setString(1, "xxxx---") - stmt_read1.setInt(2, 1231) - qe_select1 stmt_read1 - stmt_read1.setString(1, "yyyy---") - stmt_read1.setInt(2, 1232) - qe_select1 stmt_read1 - qe_select1 stmt_read1 - def stmt_read2 = prepareStatement "select * from ${tableName} as t1 join ${tableName} as t2 on t1.`k1` = t2.`k1` where t1.`k1` >= ? and t1.`k2` >= ? and size(t1.`k9`) > ? order by 1, 2, 3" - assertEquals(stmt_read2.class, com.mysql.cj.jdbc.ServerPreparedStatement); - stmt_read2.setInt(1, 1237) - stmt_read2.setBigDecimal(2, new BigDecimal("120939.11130")) - stmt_read2.setInt(3, 0) - qe_select2 stmt_read2 - qe_select2 stmt_read2 - qe_select2 stmt_read2 + def stmt_read = prepareStatement "select * from ${tableName} where k1 = ? order by k1" + assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); + stmt_read.setInt(1, 1231) + qe_select0 stmt_read + stmt_read.setInt(1, 1232) + qe_select0 stmt_read + qe_select0 stmt_read + def stmt_read1 = prepareStatement "select hex(k3), ? \n from ${tableName} where k1 = ? order by 1" + assertEquals(stmt_read1.class, com.mysql.cj.jdbc.ServerPreparedStatement); + stmt_read1.setString(1, "xxxx---") + stmt_read1.setInt(2, 1231) + qe_select1 stmt_read1 + stmt_read1.setString(1, "yyyy---") + stmt_read1.setInt(2, 1232) + qe_select1 stmt_read1 + qe_select1 stmt_read1 + def stmt_read2 = prepareStatement "select * from ${tableName} as t1 join ${tableName} as t2 on t1.`k1` = t2.`k1` where t1.`k1` >= ? and t1.`k2` >= ? and size(t1.`k9`) > ? order by 1, 2, 3" + assertEquals(stmt_read2.class, com.mysql.cj.jdbc.ServerPreparedStatement); + stmt_read2.setInt(1, 1237) + stmt_read2.setBigDecimal(2, new BigDecimal("120939.11130")) + stmt_read2.setInt(3, 0) + qe_select2 stmt_read2 + qe_select2 stmt_read2 + qe_select2 stmt_read2 - sql "DROP TABLE IF EXISTS mytable1" - sql """ - CREATE TABLE mytable1 - ( - siteid INT DEFAULT '10', - citycode SMALLINT, - username VARCHAR(32) DEFAULT '', - pv BIGINT SUM DEFAULT '0' - ) - AGGREGATE KEY(siteid, citycode, username) - DISTRIBUTED BY HASH(siteid) BUCKETS 10 - PROPERTIES("replication_num" = "1"); - """ + sql "DROP TABLE IF EXISTS mytable1" + sql """ + CREATE TABLE mytable1 + ( + siteid INT DEFAULT '10', + citycode SMALLINT, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' + ) + AGGREGATE KEY(siteid, citycode, username) + DISTRIBUTED BY HASH(siteid) BUCKETS 10 + PROPERTIES("replication_num" = "1"); + """ - sql """insert into mytable1 values(1,1,'user1',10);""" - sql """insert into mytable1 values(1,1,'user1',10);""" - sql """insert into mytable1 values(1,1,'user1',10);""" - sql "sync" - stmt_read = prepareStatement "SELECT *, ? FROM (select *, ? from mytable1 where citycode = ?) AS `SpotfireCustomQuery1` WHERE 1 = 1" - stmt_read.setInt(1, 12345) - stmt_read.setInt(2, 1234) - stmt_read.setInt(3, 1) - qe_select3 stmt_read + sql """insert into mytable1 values(1,1,'user1',10);""" + sql """insert into mytable1 values(1,1,'user1',10);""" + sql """insert into mytable1 values(1,1,'user1',10);""" + sql "sync" + stmt_read = prepareStatement "SELECT *, ? FROM (select *, ? from mytable1 where citycode = ?) AS `SpotfireCustomQuery1` WHERE 1 = 1" + assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); + stmt_read.setInt(1, 12345) + stmt_read.setInt(2, 1234) + stmt_read.setInt(3, 1) + qe_select3 stmt_read + + stmt_read = prepareStatement "SELECT 10" + assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); + qe_select4 stmt_read + stmt_read = prepareStatement "SELECT 1" + assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); + qe_select5 stmt_read + + sql """insert into mytable1 values(2,1,'user1',null);""" + stmt_read = prepareStatement "SELECT *, ? FROM (select *, ? from mytable1 where pv is null) AS `SpotfireCustomQuery1` WHERE 1 = 1" + assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); + stmt_read.setString(1, "xxxlalala") + stmt_read.setDouble(2, 1234.1111) + qe_select6 stmt_read + stmt_read.setString(1, "1111111") + stmt_read.setString(2, "1111111") + qe_select7 stmt_read + + stmt_read = prepareStatement "SELECT COUNT() from mytable1 WHERE citycode = ? GROUP BY siteid" + stmt_read.setString(1, "1") + qe_select8 stmt_read - stmt_read = prepareStatement "SELECT 10" - assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); - qe_select4 stmt_read - stmt_read = prepareStatement "SELECT 1" - assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); - qe_select5 stmt_read + stmt_read = prepareStatement "SELECT COUNT() from mytable1 WHERE citycode = ? GROUP BY ?" + stmt_read.setString(1, "1") + stmt_read.setString(2, "1") + qe_select9 stmt_read } sql "set global enable_server_side_prepared_statement = false" From 31ee3b508e7aed07a4dcff5d9a21f52d9e05ac89 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Tue, 28 May 2024 17:23:05 +0800 Subject: [PATCH 5/7] refactor --- .../org/apache/doris/nereids/DorisLexer.g4 | 2 +- .../doris/nereids/StatementContext.java | 21 +++- .../nereids/parser/LogicalPlanBuilder.java | 10 +- .../rules/analysis/ExpressionAnalyzer.java | 8 +- ...{PlaceholderExpr.java => Placeholder.java} | 46 ++++---- .../trees/expressions/literal/Literal.java | 26 ++++- .../expressions/literal/TinyIntLiteral.java | 1 - .../visitor/ExpressionVisitor.java | 6 +- .../trees/plans/commands/ExecuteCommand.java | 60 ++++++---- .../trees/plans/commands/PreparedCommand.java | 103 ++++++++++-------- .../org/apache/doris/qe/ConnectContext.java | 13 +++ .../org/apache/doris/qe/ConnectProcessor.java | 4 +- .../doris/qe/MysqlConnectProcessor.java | 71 +++++++----- .../doris/qe/PreparedStatementContext.java | 51 +++++++++ .../org/apache/doris/qe/StmtExecutor.java | 41 ++----- 15 files changed, 284 insertions(+), 179 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/{PlaceholderExpr.java => Placeholder.java} (69%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 39fae8006647fa..49c794640eb3c8 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -392,7 +392,6 @@ ORDER: 'ORDER'; OUTER: 'OUTER'; OUTFILE: 'OUTFILE'; OVER: 'OVER'; -PLACEHOLDER: '?'; OVERWRITE: 'OVERWRITE'; PARAMETER: 'PARAMETER'; PARSED: 'PARSED'; @@ -409,6 +408,7 @@ PERCENT: 'PERCENT'; PERIOD: 'PERIOD'; PERMISSIVE: 'PERMISSIVE'; PHYSICAL: 'PHYSICAL'; +PLACEHOLDER: '?'; PLAN: 'PLAN'; PROCESS: 'PROCESS'; PLUGIN: 'PLUGIN'; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 5f133d6c6220cc..2da34d2b71119d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -30,7 +30,7 @@ import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.PlaceholderExpr; +import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; @@ -120,6 +120,11 @@ public class StatementContext implements Closeable { private final Set viewDdlSqlSet = Sets.newHashSet(); private final SqlCacheContext sqlCacheContext; + // generate for next id for prepared statement's placeholders, which is connection level + private final IdGenerator placeHolderRelationIdGenerator = RelationId.createGenerator(); + // relation id to placeholders for prepared statement + private final Map idToPlaceholderRealExpr = new HashMap<>(); + // collect all hash join conditions to compute node connectivity in join graph private final List joinFilters = new ArrayList<>(); @@ -146,7 +151,7 @@ public class StatementContext implements Closeable { private final Stack plannerResources = new Stack<>(); // placeholder params for prepared statement - private List params; + private List params; // for create view support in nereids // key is the start and end position of the sql substring that needs to be replaced, @@ -374,6 +379,14 @@ public Map> getConsumerIdToFilters() { return consumerIdToFilters; } + public RelationId getNextPlaceholderId() { + return placeHolderRelationIdGenerator.getNextId(); + } + + public Map getIdToPlaceholderRealExpr() { + return idToPlaceholderRealExpr; + } + public Map, Group>>> getCteIdToConsumerGroup() { return cteIdToConsumerGroup; } @@ -494,11 +507,11 @@ public void close() { releasePlannerResources(); } - public List getParams() { + public List getParams() { return params; } - public void setParams(List params) { + public void setParams(List params) { this.params = params; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 3d0fa15c4a38e8..66ae01a8ca4ad1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -262,7 +262,7 @@ import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.OrderExpression; -import org.apache.doris.nereids.trees.expressions.PlaceholderExpr; +import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.Regexp; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; @@ -495,11 +495,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { private final boolean forCreateView; - private int placeHolderExprId = 0; - // Sort the parameters with token position to keep the order with original placeholders // in prepared statement.Otherwise, the order maybe broken - private Map tokenPosToParameters; + private Map tokenPosToParameters; public LogicalPlanBuilder() { forCreateView = false; @@ -1018,7 +1016,7 @@ public List> visitMultiStatements(MultiState logicalPlans.add(Pair.of( ParserUtils.withOrigin(ctx, () -> (LogicalPlan) visit(statement)), statementContext)); if (tokenPosToParameters != null) { - List params = new ArrayList<>(tokenPosToParameters.values()); + List params = new ArrayList<>(tokenPosToParameters.values()); statementContext.setParams(params); tokenPosToParameters.clear(); } @@ -2345,7 +2343,7 @@ public Expression visitPlaceholder(DorisParser.PlaceholderContext ctx) { return pos1.getCharPositionInLine() - pos2.getCharPositionInLine(); }); } - PlaceholderExpr parameter = new PlaceholderExpr(placeHolderExprId++); + Placeholder parameter = new Placeholder(ConnectContext.get().getStatementContext().getNextPlaceholderId()); tokenPosToParameters.put(ctx.start, parameter); return parameter; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index c662c84bc7e988..0cbb8d7fa2087f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -56,7 +56,7 @@ import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; -import org.apache.doris.nereids.trees.expressions.PlaceholderExpr; +import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; @@ -543,8 +543,10 @@ public Expression visitNot(Not not, ExpressionRewriteContext context) { } @Override - public Expression visitPlaceholderExpr(PlaceholderExpr placeholderExpr, ExpressionRewriteContext context) { - return visit(placeholderExpr.getExpr(), context); + public Expression visitPlaceholderExpr(Placeholder placeholderExpr, ExpressionRewriteContext context) { + Expression realExpr = context.cascadesContext.getStatementContext() + .getIdToPlaceholderRealExpr().get(placeholderExpr.getExprId()); + return visit(realExpr, context); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java similarity index 69% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java index e0dd67357b147b..41b398c0d43241 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PlaceholderExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java @@ -20,54 +20,38 @@ import org.apache.doris.catalog.MysqlColType; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.types.DataType; /** * Placeholder for prepared statement */ -public class PlaceholderExpr extends Expression { - private final int exprId; - private Expression expr; +public class Placeholder extends Expression { + private final RelationId exprId; + private final MysqlColType mysqlColType; - private MysqlColType mysqlTypeCode; - - public PlaceholderExpr(int exprId) { + public Placeholder(RelationId exprId) { this.exprId = exprId; + this.mysqlColType = null; } - protected PlaceholderExpr(int exprId, Expression expr) { + public Placeholder(RelationId exprId, MysqlColType mysqlColType) { this.exprId = exprId; - this.expr = expr; - } - - public Expression getExpr() { - return expr; + this.mysqlColType = mysqlColType; } - public int getExprId() { + public RelationId getExprId() { return exprId; } - public void setExpr(Expression expr) { - this.expr = expr; - } - @Override public R accept(ExpressionVisitor visitor, C context) { return visitor.visitPlaceholderExpr(this, context); } - public void setTypeCode(int mysqlTypeCode) { - this.mysqlTypeCode = MysqlColType.fromCode(mysqlTypeCode); - } - - public MysqlColType getMysqlTypeCode() { - return mysqlTypeCode; - } - @Override public boolean nullable() { - return expr.nullable(); + return true; } @Override @@ -77,6 +61,14 @@ public String toSql() { @Override public DataType getDataType() throws UnboundException { - return expr.getDataType(); + return null; + } + + public Placeholder withNewMysqlColType(MysqlColType mysqlColType) { + return new Placeholder(getExprId(), mysqlColType); + } + + public MysqlColType getMysqlColType() { + return mysqlColType; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 8ae54b4418a2bf..9c1096799baeae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -451,6 +451,7 @@ public static int getParmLen(ByteBuffer data) { * @param data the ByteBuffer containing the data * @return a Literal object corresponding to the MySQL type * @throws AnalysisException if the MySQL type is unsupported or if data conversion fails + * @link .... */ public static Literal getLiteralByMysqlType(MysqlColType mysqlType, ByteBuffer data) throws AnalysisException { switch (mysqlType) { @@ -485,32 +486,41 @@ public static Literal getLiteralByMysqlType(MysqlColType mysqlType, ByteBuffer d } } - private static DecimalLiteral handleDecimalLiteral(ByteBuffer data) throws AnalysisException { + private static Literal handleDecimalLiteral(ByteBuffer data) throws AnalysisException { int len = getParmLen(data); byte[] bytes = new byte[len]; data.get(bytes); try { String value = new String(bytes); BigDecimal v = new BigDecimal(value); + if (Config.enable_decimal_conversion) { + return new DecimalV3Literal(v); + } return new DecimalLiteral(v); } catch (NumberFormatException e) { throw new AnalysisException("Invalid decimal literal", e); } } - private static DateLiteral handleDateLiteral(ByteBuffer data) { + private static Literal handleDateLiteral(ByteBuffer data) { int len = getParmLen(data); if (len >= 4) { int year = (int) data.getChar(); int month = (int) data.get(); int day = (int) data.get(); + if (Config.enable_date_conversion) { + return new DateV2Literal(year, month, day); + } return new DateLiteral(year, month, day); } else { + if (Config.enable_date_conversion) { + return new DateV2Literal(0, 1, 1); + } return new DateLiteral(0, 1, 1); } } - private static DateTimeLiteral handleDateTimeLiteral(ByteBuffer data) { + private static Literal handleDateTimeLiteral(ByteBuffer data) { int len = getParmLen(data); if (len >= 4) { int year = (int) data.getChar(); @@ -528,13 +538,19 @@ private static DateTimeLiteral handleDateTimeLiteral(ByteBuffer data) { if (len > 7) { microsecond = data.getInt(); } + if (Config.enable_date_conversion) { + return new DateTimeV2Literal(year, month, day, hour, minute, second, microsecond); + } return new DateTimeLiteral(DateTimeType.INSTANCE, year, month, day, hour, minute, second, microsecond); } else { + if (Config.enable_date_conversion) { + return new DateTimeV2Literal(0, 1, 1, 0, 0, 0); + } return new DateTimeLiteral(0, 1, 1, 0, 0, 0); } } - private static StringLiteral handleStringLiteral(ByteBuffer data) { + private static Literal handleStringLiteral(ByteBuffer data) { int strLen = getParmLen(data); strLen = Math.min(strLen, data.remaining()); byte[] bytes = new byte[strLen]; @@ -542,7 +558,7 @@ private static StringLiteral handleStringLiteral(ByteBuffer data) { return new StringLiteral(new String(bytes)); } - private static VarcharLiteral handleVarcharLiteral(ByteBuffer data) { + private static Literal handleVarcharLiteral(ByteBuffer data) { int strLen = getParmLen(data); strLen = Math.min(strLen, data.remaining()); byte[] bytes = new byte[strLen]; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TinyIntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TinyIntLiteral.java index feb1ba80c95adf..08d46c0e5ca69a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TinyIntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TinyIntLiteral.java @@ -65,5 +65,4 @@ public int getIntValue() { public Number getNumber() { return value; } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index b6be307fde4482..485b5d58c7e6e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -69,7 +69,7 @@ import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.OrderExpression; -import org.apache.doris.nereids.trees.expressions.PlaceholderExpr; +import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.Properties; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.Slot; @@ -506,8 +506,8 @@ public R visitMatchPhraseEdge(MatchPhraseEdge matchPhraseEdge, C context) { return visitMatch(matchPhraseEdge, context); } - public R visitPlaceholderExpr(PlaceholderExpr placeholderExpr, C context) { - return null; + public R visitPlaceholderExpr(Placeholder placeholderExpr, C context) { + return visit(placeholderExpr, context); } public R visitAny(Any any, C context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java index 209a0bc288f911..e4d0174e16b8ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java @@ -17,12 +17,15 @@ package org.apache.doris.nereids.trees.plans.commands; -import org.apache.doris.analysis.RedirectStatus; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.PreparedStatementContext; +import org.apache.doris.qe.StmtExecutor; import java.util.List; import java.util.stream.Collectors; @@ -30,20 +33,16 @@ /** * Prepared Statement */ -public class ExecuteCommand extends LogicalPlanAdapter { +public class ExecuteCommand extends Command { private final String stmtName; - private final List params; + private final PreparedCommand preparedCommand; + private final StatementContext statementContext; - private StatementContext statementContext; - - public ExecuteCommand(String stmtName, List paramsExpr) { - super(null, null); + public ExecuteCommand(String stmtName, PreparedCommand preparedCommand, StatementContext statementContext) { + super(PlanType.UNKNOWN); this.stmtName = stmtName; - this.params = paramsExpr; - } - - public List getParams() { - return params; + this.preparedCommand = preparedCommand; + this.statementContext = statementContext; } public String getStmtName() { @@ -51,21 +50,34 @@ public String getStmtName() { } @Override - public String toSql() { - return "EXECUTE `" + stmtName + "`" - + params.stream().map(Expression::toSql).collect(Collectors.joining(", ", " USING ", "")); + public R accept(PlanVisitor visitor, C context) { + return visitor.visit(this, context); } @Override - public RedirectStatus getRedirectStatus() { - return null; + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + PreparedStatementContext preparedStmtCtx = ctx.getPreparedStementContext(stmtName); + if (null == preparedStmtCtx) { + throw new AnalysisException( + "prepare statement " + stmtName + " not found, maybe expired"); + } + PreparedCommand preparedCommand = (PreparedCommand) preparedStmtCtx.command; + LogicalPlanAdapter planAdapter = new LogicalPlanAdapter(preparedCommand.getInnerPlan(), executor.getContext() + .getStatementContext()); + executor.setParsedStmt(planAdapter); + // execute real statement + executor.execute(); } - @Override - public StatementContext getStatementContext() { - if (statementContext == null) { - statementContext = new StatementContext(ConnectContext.get(), new OriginStatement("", 0)); - } - return statementContext; + /** + * return the sql representation contains real expr instead of placeholders + */ + public String toSql() { + // maybe slow + List realValueExpr = preparedCommand.params().stream() + .map(placeholder -> statementContext.getIdToPlaceholderRealExpr().get(placeholder.getExprId())) + .collect(Collectors.toList()); + return "EXECUTE `" + stmtName + "`" + + realValueExpr.stream().map(Expression::toSql).collect(Collectors.joining(", ", " USING ", "")); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java index 19abb104fb2609..8fa1fe1db2bf65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java @@ -17,13 +17,18 @@ package org.apache.doris.nereids.trees.plans.commands; -import org.apache.doris.analysis.RedirectStatus; -import org.apache.doris.analysis.StatementBase; -import org.apache.doris.nereids.glue.LogicalPlanAdapter; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.PlaceholderExpr; - -import com.google.common.base.Preconditions; +import org.apache.doris.mysql.MysqlCommand; +import org.apache.doris.nereids.trees.expressions.Placeholder; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.PreparedStatementContext; +import org.apache.doris.qe.StmtExecutor; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.List; @@ -31,28 +36,39 @@ /** * Prepared Statement */ -public class PreparedCommand extends LogicalPlanAdapter { - protected List params; - private final StatementBase inner; +public class PreparedCommand extends Command { + private static final Logger LOG = LogManager.getLogger(StmtExecutor.class); + + protected List params = new ArrayList<>(); + private final LogicalPlan inner; + + private final String stmtName; - private String stmtName; + private final OriginStatement originalStmt; - public PreparedCommand(String name, StatementBase originStmt, List placeholders) { - super(null, null); - this.inner = originStmt; - this.params = placeholders; + /** + * constructor + * @param name the statement name which represents statement id for prepared statement + * @param plan the inner statement + * @param placeholders the parameters for this prepared statement + * @param originalStmt original statement from StmtExecutor + */ + public PreparedCommand(String name, LogicalPlan plan, List placeholders, + OriginStatement originalStmt) { + super(PlanType.UNKNOWN); + this.inner = plan; + if (placeholders != null) { + this.params = placeholders; + } this.stmtName = name; + this.originalStmt = originalStmt; } public String getName() { return stmtName; } - public void setName(String name) { - this.stmtName = name; - } - - public List params() { + public List params() { return params; } @@ -63,26 +79,12 @@ public int getParamLen() { return params.size(); } - public void setParams(List params) { - this.params = params; - } - - public StatementBase getInnerStmt() { + public LogicalPlan getInnerPlan() { return inner; } - /** - * assign real value to placeholders and return the assigned statement - */ - public StatementBase assignValues(List values) { - if (params == null) { - return inner; - } - Preconditions.checkArgument(values.size() == params.size(), "Invalid arguments size"); - for (int i = 0; i < params.size(); i++) { - params.get(i).setExpr(values.get(i)); - } - return inner; + public OriginStatement getOriginalStmt() { + return originalStmt; } /** @@ -93,19 +95,34 @@ public List getLabels() { if (params == null) { return labels; } - for (PlaceholderExpr parameter : params) { - labels.add("$" + parameter.getExprId()); + for (Placeholder parameter : params) { + labels.add("$" + parameter.getExprId().asInt()); } return labels; } + // register prepared statement with attached statement id @Override - public String toSql() { - return "PREPARE " + stmtName + " FROM " + inner.toSql(); + public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + List labels = getLabels(); + // register prepareStmt + if (LOG.isDebugEnabled()) { + LOG.debug("add prepared statement {}, isBinaryProtocol {}", + stmtName, ctx.getCommand() == MysqlCommand.COM_STMT_PREPARE); + } + ctx.addPreparedStatementContext(stmtName, + new PreparedStatementContext(this, ctx, ctx.getStatementContext(), stmtName)); + if (ctx.getCommand() == MysqlCommand.COM_STMT_PREPARE) { + executor.sendStmtPrepareOK((int) ctx.getStmtId(), labels); + } } @Override - public RedirectStatus getRedirectStatus() { - return inner.getRedirectStatus(); + public R accept(PlanVisitor visitor, C context) { + return visitor.visit(this, context); + } + + public PreparedCommand withNewPreparedCommand(List params) { + return new PreparedCommand(this.stmtName, this.inner, params, this.originalStmt); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index f970d9b406e824..b166de7691c08d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -251,8 +251,13 @@ public void setUserInsertTimeout(int insertTimeout) { } private StatementContext statementContext; + + // legacy planner private Map preparedStmtCtxs = Maps.newHashMap(); + // new planner + private Map preparedStatementContextMap = Maps.newHashMap(); + private List tables = null; private Map totalColumnStatisticMap = new HashMap<>(); @@ -385,6 +390,10 @@ public void addPreparedStmt(String stmtName, PrepareStmtContext ctx) { this.preparedStmtCtxs.put(stmtName, ctx); } + public void addPreparedStatementContext(String stmtName, PreparedStatementContext ctx) { + this.preparedStatementContextMap.put(stmtName, ctx); + } + public void removePrepareStmt(String stmtName) { this.preparedStmtCtxs.remove(stmtName); } @@ -393,6 +402,10 @@ public PrepareStmtContext getPreparedStmt(String stmtName) { return this.preparedStmtCtxs.get(stmtName); } + public PreparedStatementContext getPreparedStementContext(String stmtName) { + return this.preparedStatementContextMap.get(stmtName); + } + public List getTables() { return tables; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index f6fdab98ef6c0a..7d7e006d9d4228 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -257,7 +257,9 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex Exception nereidsSyntaxException = null; long parseSqlStartTime = System.currentTimeMillis(); List cachedStmts = null; - if (sessionVariable.isEnableNereidsPlanner()) { + boolean nereidsUseServerPrep = sessionVariable.enableServeSidePreparedStatement + || mysqlCommand == MysqlCommand.COM_QUERY; + if (nereidsUseServerPrep && sessionVariable.isEnableNereidsPlanner()) { if (wantToParseSqlFromSqlCache) { cachedStmts = parseFromSqlCache(originStmt); if (cachedStmts != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index c3c127e389a697..3df4816d475f7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -22,13 +22,19 @@ import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.PrepareStmt; import org.apache.doris.analysis.QueryStmt; +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.catalog.MysqlColType; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.mysql.MysqlProto; -import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.ExecuteCommand; import org.apache.doris.nereids.trees.plans.commands.PreparedCommand; @@ -143,46 +149,53 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) { auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); } - private void handleExecute(PreparedCommand prepareStmt, long stmtId) { - prepareStmt.setIsPrepared(); + private void handleExecute(Command preparedCommand, long stmtId, PreparedStatementContext prepCtx) { + PreparedCommand prepareStmt = (PreparedCommand) preparedCommand; int paramCount = prepareStmt.getParamLen(); LOG.debug("execute prepared statement {}, paramCount {}", stmtId, paramCount); // null bitmap String stmtStr = ""; try { - List realValueExprs = new ArrayList<>(); + StatementContext statementContext = prepCtx.statementContext; if (paramCount > 0) { byte[] nullbitmapData = new byte[(paramCount + 7) / 8]; packetBuf.get(nullbitmapData); // new_params_bind_flag if ((int) packetBuf.get() != 0) { + List typedPlaceholders = new ArrayList<>(); // parse params's types for (int i = 0; i < paramCount; ++i) { int typeCode = packetBuf.getChar(); LOG.debug("code {}", typeCode); - prepareStmt.params().get(i).setTypeCode(typeCode); + // assign type to placeholders + typedPlaceholders.add( + prepareStmt.params().get(i).withNewMysqlColType(MysqlColType.fromCode(typeCode))); } + // rewrite with new prepared statment with type info in placeholders + prepCtx.command = prepareStmt.withNewPreparedCommand(typedPlaceholders); + prepareStmt = (PreparedCommand) prepCtx.command; } // parse param data for (int i = 0; i < paramCount; ++i) { + RelationId exprId = prepareStmt.params().get(i).getExprId(); if (isNull(nullbitmapData, i)) { - realValueExprs.add(new org.apache.doris.nereids.trees.expressions.literal.NullLiteral()); + statementContext.getIdToPlaceholderRealExpr().put(exprId, + new org.apache.doris.nereids.trees.expressions.literal.NullLiteral()); continue; } - Literal l = Literal.getLiteralByMysqlType( - prepareStmt.params().get(i).getMysqlTypeCode(), packetBuf); - realValueExprs.add(l); + MysqlColType type = prepareStmt.params().get(i).getMysqlColType(); + Literal l = Literal.getLiteralByMysqlType(type, packetBuf); + statementContext.getIdToPlaceholderRealExpr().put(exprId, l); } } - ExecuteCommand executeStmt = new ExecuteCommand(String.valueOf(stmtId), realValueExprs); + ExecuteCommand executeStmt = new ExecuteCommand(String.valueOf(stmtId), prepareStmt, statementContext); // TODO set real origin statement - executeStmt.setOrigStmt(new OriginStatement("null", 0)); - executeStmt.setUserInfo(ctx.getCurrentUserIdentity()); if (LOG.isDebugEnabled()) { LOG.debug("executeStmt {}", executeStmt); } - executeStmt.setOrigStmt(prepareStmt.getInnerStmt().getOrigStmt()); - executor = new StmtExecutor(ctx, executeStmt); + StatementBase stmt = new LogicalPlanAdapter(executeStmt, statementContext); + stmt.setOrigStmt(prepareStmt.getOriginalStmt()); + executor = new StmtExecutor(ctx, stmt); ctx.setExecutor(executor); executor.execute(); stmtStr = executeStmt.toSql(); @@ -210,26 +223,24 @@ private void handleExecute() { if (LOG.isDebugEnabled()) { LOG.debug("execute prepared statement {}", stmtId); } + PrepareStmtContext prepareCtx = ctx.getPreparedStmt(String.valueOf(stmtId)); - if (prepareCtx == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("No such statement in context, stmtId:{}", stmtId); - } - ctx.getState().setError(ErrorCode.ERR_UNKNOWN_COM_ERROR, - "msg: Not supported such prepared statement"); - return; - } ctx.setStartTime(); - if (prepareCtx.stmt instanceof PrepareStmt) { - // legacy + if (prepareCtx != null) { + // get from lagacy planner context, to be removed handleExecute((PrepareStmt) prepareCtx.stmt, stmtId); - } else if (prepareCtx.stmt instanceof PreparedCommand) { - // nererids - handleExecute((PreparedCommand) prepareCtx.stmt, stmtId); } else { - ctx.getState().setError(ErrorCode.ERR_UNKNOWN_COM_ERROR, - "msg: Not supported such prepared statement"); - return; + // nererids + PreparedStatementContext preparedStatementContext = ctx.getPreparedStementContext(String.valueOf(stmtId)); + if (preparedStatementContext == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("No such statement in context, stmtId:{}", stmtId); + } + ctx.getState().setError(ErrorCode.ERR_UNKNOWN_COM_ERROR, + "msg: Not supported such prepared statement"); + return; + } + handleExecute(preparedStatementContext.command, stmtId, preparedStatementContext); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java new file mode 100644 index 00000000000000..d50a865bd70269 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.commands.Command; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class PreparedStatementContext { + private static final Logger LOG = LogManager.getLogger(PrepareStmtContext.class); + public Command command; + public ConnectContext ctx; + StatementContext statementContext; + public String stmtString; + + // Timestamp in millisecond last command starts at + protected volatile long startTime; + + public PreparedStatementContext(Command command, + ConnectContext ctx, StatementContext statementContext, String stmtString) { + this.command = command; + this.ctx = ctx; + this.statementContext = statementContext; + this.stmtString = stmtString; + } + + public long getStartTime() { + return startTime; + } + + public void setStartTime() { + startTime = System.currentTimeMillis(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 68b4116827cece..52be94fe49cf81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -150,7 +150,6 @@ import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromUsingCommand; -import org.apache.doris.nereids.trees.plans.commands.ExecuteCommand; import org.apache.doris.nereids.trees.plans.commands.Forward; import org.apache.doris.nereids.trees.plans.commands.NotAllowFallback; import org.apache.doris.nereids.trees.plans.commands.PreparedCommand; @@ -682,6 +681,14 @@ private void executeByNereids(TUniqueId queryId) throws Exception { "Nereids only process LogicalPlanAdapter, but parsedStmt is " + parsedStmt.getClass().getName()); context.getState().setNereids(true); LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { + if (isForwardToMaster()) { + throw new UserException("Forward master command is not supported for prepare statement"); + } + logicalPlan = new PreparedCommand(String.valueOf(context.getStmtId()), + logicalPlan, statementContext.getParams(), originStmt); + + } // when we in transaction mode, we only support insert into command and transaction command if (context.isTxnModel()) { if (!(logicalPlan instanceof BatchInsertIntoTableCommand || logicalPlan instanceof InsertIntoTableCommand @@ -692,25 +699,6 @@ private void executeByNereids(TUniqueId queryId) throws Exception { throw new NereidsException(errMsg, new AnalysisException(errMsg)); } } - if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { - if (isForwardToMaster()) { - throw new UserException("Forward master command is not supported for prepare statement"); - } - // register statement and return - handlePrepareStmt(); - return; - } - if (context.getCommand() == MysqlCommand.COM_STMT_EXECUTE) { - ExecuteCommand executeStmt = (ExecuteCommand) parsedStmt; - preparedStmtCtx = context.getPreparedStmt(executeStmt.getStmtName()); - if (null == preparedStmtCtx) { - throw new AnalysisException( - "prepare statement " + executeStmt.getStmtName() + " not found, maybe expired"); - } - PreparedCommand prepareStmt = (PreparedCommand) preparedStmtCtx.stmt; - parsedStmt = prepareStmt.assignValues(executeStmt.getParams()); - logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); - } if (logicalPlan instanceof Command) { if (logicalPlan instanceof Forward) { redirectStatus = ((Forward) logicalPlan).toRedirectStatus(); @@ -2606,16 +2594,7 @@ private void handleSwitchStmt() throws AnalysisException { } private void handlePrepareStmt() throws Exception { - List labels; - if (prepareStmt == null) { - // must be in nereids, todo remove original planner code - prepareStmt = new PreparedCommand(String.valueOf(context.getStmtId()), - parsedStmt, statementContext.getParams()); - labels = ((PreparedCommand) prepareStmt).getLabels(); - } else { - // legacy planner - labels = ((PrepareStmt) prepareStmt).getColLabelsOfPlaceHolders(); - } + List labels = ((PrepareStmt) prepareStmt).getColLabelsOfPlaceHolders(); // register prepareStmt if (LOG.isDebugEnabled()) { LOG.debug("add prepared statement {}, isBinaryProtocol {}", @@ -2729,7 +2708,7 @@ private List exprToStringType(List exprs) { return exprs.stream().map(e -> PrimitiveType.STRING).collect(Collectors.toList()); } - private void sendStmtPrepareOK(int stmtId, List labels) throws IOException { + public void sendStmtPrepareOK(int stmtId, List labels) throws IOException { Preconditions.checkState(context.getConnectType() == ConnectType.MYSQL); // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_prepare.html#sect_protocol_com_stmt_prepare_response serializer.reset(); From c3c8244e12b0a470316808778af77268c8cbef0a Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 30 May 2024 19:07:03 +0800 Subject: [PATCH 6/7] modify according to cr --- .../doris/nereids/StatementContext.java | 21 +++---- .../nereids/parser/LogicalPlanBuilder.java | 25 ++++---- .../rules/analysis/ExpressionAnalyzer.java | 2 +- .../trees/expressions/Placeholder.java | 28 +++++---- .../visitor/ExpressionVisitor.java | 2 +- .../nereids/trees/plans/PlaceholderId.java | 58 +++++++++++++++++++ .../doris/nereids/trees/plans/PlanType.java | 5 +- .../trees/plans/commands/ExecuteCommand.java | 14 ++--- ...eparedCommand.java => PrepareCommand.java} | 48 +++++++-------- .../org/apache/doris/qe/ConnectProcessor.java | 2 + .../doris/qe/MysqlConnectProcessor.java | 24 ++++---- .../doris/qe/PreparedStatementContext.java | 10 +--- .../org/apache/doris/qe/StmtExecutor.java | 6 +- 13 files changed, 147 insertions(+), 98 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceholderId.java rename fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{PreparedCommand.java => PrepareCommand.java} (74%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 2da34d2b71119d..ab8b99c51a3be4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.PlaceholderId; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.TableId; import org.apache.doris.nereids.trees.plans.algebra.Relation; @@ -121,9 +122,9 @@ public class StatementContext implements Closeable { private final SqlCacheContext sqlCacheContext; // generate for next id for prepared statement's placeholders, which is connection level - private final IdGenerator placeHolderRelationIdGenerator = RelationId.createGenerator(); + private final IdGenerator placeHolderIdGenerator = PlaceholderId.createGenerator(); // relation id to placeholders for prepared statement - private final Map idToPlaceholderRealExpr = new HashMap<>(); + private final Map idToPlaceholderRealExpr = new HashMap<>(); // collect all hash join conditions to compute node connectivity in join graph private final List joinFilters = new ArrayList<>(); @@ -151,7 +152,7 @@ public class StatementContext implements Closeable { private final Stack plannerResources = new Stack<>(); // placeholder params for prepared statement - private List params; + private List placeholders; // for create view support in nereids // key is the start and end position of the sql substring that needs to be replaced, @@ -379,11 +380,11 @@ public Map> getConsumerIdToFilters() { return consumerIdToFilters; } - public RelationId getNextPlaceholderId() { - return placeHolderRelationIdGenerator.getNextId(); + public PlaceholderId getNextPlaceholderId() { + return placeHolderIdGenerator.getNextId(); } - public Map getIdToPlaceholderRealExpr() { + public Map getIdToPlaceholderRealExpr() { return idToPlaceholderRealExpr; } @@ -507,12 +508,12 @@ public void close() { releasePlannerResources(); } - public List getParams() { - return params; + public List getPlaceholders() { + return placeholders; } - public void setParams(List params) { - this.params = params; + public void setPlaceholders(List placeholders) { + this.placeholders = placeholders; } private static class CloseableResource implements Closeable { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 66ae01a8ca4ad1..5ac16f926844e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -497,7 +497,13 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor { // Sort the parameters with token position to keep the order with original placeholders // in prepared statement.Otherwise, the order maybe broken - private Map tokenPosToParameters; + private final Map tokenPosToParameters = Maps.newTreeMap((pos1, pos2) -> { + int line = pos1.getLine() - pos2.getLine(); + if (line != 0) { + return line; + } + return pos1.getCharPositionInLine() - pos2.getCharPositionInLine(); + }); public LogicalPlanBuilder() { forCreateView = false; @@ -1015,11 +1021,9 @@ public List> visitMultiStatements(MultiState } logicalPlans.add(Pair.of( ParserUtils.withOrigin(ctx, () -> (LogicalPlan) visit(statement)), statementContext)); - if (tokenPosToParameters != null) { - List params = new ArrayList<>(tokenPosToParameters.values()); - statementContext.setParams(params); - tokenPosToParameters.clear(); - } + List params = new ArrayList<>(tokenPosToParameters.values()); + statementContext.setPlaceholders(params); + tokenPosToParameters.clear(); } return logicalPlans; } @@ -2334,15 +2338,6 @@ public Literal visitStringLiteral(StringLiteralContext ctx) { @Override public Expression visitPlaceholder(DorisParser.PlaceholderContext ctx) { - if (tokenPosToParameters == null) { - tokenPosToParameters = Maps.newTreeMap((pos1, pos2) -> { - int line = pos1.getLine() - pos2.getLine(); - if (line != 0) { - return line; - } - return pos1.getCharPositionInLine() - pos2.getCharPositionInLine(); - }); - } Placeholder parameter = new Placeholder(ConnectContext.get().getStatementContext().getNextPlaceholderId()); tokenPosToParameters.put(ctx.start, parameter); return parameter; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index 0cbb8d7fa2087f..e8f580e5fb0d98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -543,7 +543,7 @@ public Expression visitNot(Not not, ExpressionRewriteContext context) { } @Override - public Expression visitPlaceholderExpr(Placeholder placeholderExpr, ExpressionRewriteContext context) { + public Expression visitPlaceholder(Placeholder placeholderExpr, ExpressionRewriteContext context) { Expression realExpr = context.cascadesContext.getStatementContext() .getIdToPlaceholderRealExpr().get(placeholderExpr.getExprId()); return visit(realExpr, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java index 41b398c0d43241..061360411d9864 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java @@ -19,34 +19,38 @@ import org.apache.doris.catalog.MysqlColType; import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.trees.expressions.shape.LeafExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.PlaceholderId; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.NullType; + +import java.util.Optional; /** * Placeholder for prepared statement */ -public class Placeholder extends Expression { - private final RelationId exprId; - private final MysqlColType mysqlColType; +public class Placeholder extends Expression implements LeafExpression { + private final PlaceholderId exprId; + private final Optional mysqlColType; - public Placeholder(RelationId exprId) { + public Placeholder(PlaceholderId exprId) { this.exprId = exprId; - this.mysqlColType = null; + this.mysqlColType = Optional.empty(); } - public Placeholder(RelationId exprId, MysqlColType mysqlColType) { + public Placeholder(PlaceholderId exprId, MysqlColType mysqlColType) { this.exprId = exprId; - this.mysqlColType = mysqlColType; + this.mysqlColType = Optional.of(mysqlColType); } - public RelationId getExprId() { + public PlaceholderId getExprId() { return exprId; } @Override public R accept(ExpressionVisitor visitor, C context) { - return visitor.visitPlaceholderExpr(this, context); + return visitor.visitPlaceholder(this, context); } @Override @@ -61,7 +65,7 @@ public String toSql() { @Override public DataType getDataType() throws UnboundException { - return null; + return NullType.INSTANCE; } public Placeholder withNewMysqlColType(MysqlColType mysqlColType) { @@ -69,6 +73,6 @@ public Placeholder withNewMysqlColType(MysqlColType mysqlColType) { } public MysqlColType getMysqlColType() { - return mysqlColType; + return mysqlColType.get(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index 485b5d58c7e6e7..7a7a60f9c37cca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -506,7 +506,7 @@ public R visitMatchPhraseEdge(MatchPhraseEdge matchPhraseEdge, C context) { return visitMatch(matchPhraseEdge, context); } - public R visitPlaceholderExpr(Placeholder placeholderExpr, C context) { + public R visitPlaceholder(Placeholder placeholderExpr, C context) { return visit(placeholderExpr, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceholderId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceholderId.java new file mode 100644 index 00000000000000..f1d410100e16fa --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlaceholderId.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.common.Id; +import org.apache.doris.common.IdGenerator; + +/** + * placeholder id for prepared statement parameters + */ +public class PlaceholderId extends Id { + + public PlaceholderId(int id) { + super(id); + } + + /** + * Should be only called by {@link org.apache.doris.nereids.StatementContext}. + */ + public static IdGenerator createGenerator() { + return new IdGenerator() { + @Override + public PlaceholderId getNextId() { + return new PlaceholderId(nextId++); + } + }; + } + + @Override + public String toString() { + return "PlaceholderId#" + id; + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 210bab09c676cf..13fd64a8798851 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -157,5 +157,8 @@ public enum PlanType { ALTER_VIEW_COMMAND, DROP_CATALOG_RECYCLE_BIN_COMMAND, UNSUPPORTED_COMMAND, - CREATE_TABLE_LIKE_COMMAND + CREATE_TABLE_LIKE_COMMAND, + + PREPARED_COMMAND, + EXECUTE_COMMAND } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java index e4d0174e16b8ad..18925ae7756fbf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java @@ -35,13 +35,13 @@ */ public class ExecuteCommand extends Command { private final String stmtName; - private final PreparedCommand preparedCommand; + private final PrepareCommand prepareCommand; private final StatementContext statementContext; - public ExecuteCommand(String stmtName, PreparedCommand preparedCommand, StatementContext statementContext) { - super(PlanType.UNKNOWN); + public ExecuteCommand(String stmtName, PrepareCommand prepareCommand, StatementContext statementContext) { + super(PlanType.EXECUTE_COMMAND); this.stmtName = stmtName; - this.preparedCommand = preparedCommand; + this.prepareCommand = prepareCommand; this.statementContext = statementContext; } @@ -61,8 +61,8 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { throw new AnalysisException( "prepare statement " + stmtName + " not found, maybe expired"); } - PreparedCommand preparedCommand = (PreparedCommand) preparedStmtCtx.command; - LogicalPlanAdapter planAdapter = new LogicalPlanAdapter(preparedCommand.getInnerPlan(), executor.getContext() + PrepareCommand prepareCommand = (PrepareCommand) preparedStmtCtx.command; + LogicalPlanAdapter planAdapter = new LogicalPlanAdapter(prepareCommand.getInnerPlan(), executor.getContext() .getStatementContext()); executor.setParsedStmt(planAdapter); // execute real statement @@ -74,7 +74,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { */ public String toSql() { // maybe slow - List realValueExpr = preparedCommand.params().stream() + List realValueExpr = prepareCommand.params().stream() .map(placeholder -> statementContext.getIdToPlaceholderRealExpr().get(placeholder.getExprId())) .collect(Collectors.toList()); return "EXECUTE `" + stmtName + "`" diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java similarity index 74% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java index 8fa1fe1db2bf65..8629ef4e4767ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PreparedCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java @@ -36,13 +36,13 @@ /** * Prepared Statement */ -public class PreparedCommand extends Command { +public class PrepareCommand extends Command { private static final Logger LOG = LogManager.getLogger(StmtExecutor.class); - protected List params = new ArrayList<>(); - private final LogicalPlan inner; + private final List placeholders = new ArrayList<>(); + private final LogicalPlan logicalPlan; - private final String stmtName; + private final String name; private final OriginStatement originalStmt; @@ -53,34 +53,29 @@ public class PreparedCommand extends Command { * @param placeholders the parameters for this prepared statement * @param originalStmt original statement from StmtExecutor */ - public PreparedCommand(String name, LogicalPlan plan, List placeholders, + public PrepareCommand(String name, LogicalPlan plan, List placeholders, OriginStatement originalStmt) { - super(PlanType.UNKNOWN); - this.inner = plan; - if (placeholders != null) { - this.params = placeholders; - } - this.stmtName = name; + super(PlanType.PREPARED_COMMAND); + this.logicalPlan = plan; + this.placeholders.addAll(placeholders); + this.name = name; this.originalStmt = originalStmt; } public String getName() { - return stmtName; + return name; } public List params() { - return params; + return placeholders; } public int getParamLen() { - if (params == null) { - return 0; - } - return params.size(); + return placeholders.size(); } public LogicalPlan getInnerPlan() { - return inner; + return logicalPlan; } public OriginStatement getOriginalStmt() { @@ -88,14 +83,11 @@ public OriginStatement getOriginalStmt() { } /** - * return the labels of paramters + * return the labels of parameters */ public List getLabels() { List labels = new ArrayList<>(); - if (params == null) { - return labels; - } - for (Placeholder parameter : params) { + for (Placeholder parameter : placeholders) { labels.add("$" + parameter.getExprId().asInt()); } return labels; @@ -108,10 +100,10 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { // register prepareStmt if (LOG.isDebugEnabled()) { LOG.debug("add prepared statement {}, isBinaryProtocol {}", - stmtName, ctx.getCommand() == MysqlCommand.COM_STMT_PREPARE); + name, ctx.getCommand() == MysqlCommand.COM_STMT_PREPARE); } - ctx.addPreparedStatementContext(stmtName, - new PreparedStatementContext(this, ctx, ctx.getStatementContext(), stmtName)); + ctx.addPreparedStatementContext(name, + new PreparedStatementContext(this, ctx, ctx.getStatementContext(), name)); if (ctx.getCommand() == MysqlCommand.COM_STMT_PREPARE) { executor.sendStmtPrepareOK((int) ctx.getStmtId(), labels); } @@ -122,7 +114,7 @@ public R accept(PlanVisitor visitor, C context) { return visitor.visit(this, context); } - public PreparedCommand withNewPreparedCommand(List params) { - return new PreparedCommand(this.stmtName, this.inner, params, this.originalStmt); + public PrepareCommand withPlaceholders(List placeholders) { + return new PrepareCommand(this.name, this.logicalPlan, placeholders, this.originalStmt); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 7d7e006d9d4228..d7053e9631a1ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -257,6 +257,8 @@ public void executeQuery(MysqlCommand mysqlCommand, String originStmt) throws Ex Exception nereidsSyntaxException = null; long parseSqlStartTime = System.currentTimeMillis(); List cachedStmts = null; + // Currently we add a config to decide whether using PREPARED/EXECUTE command for nereids + // TODO: after implemented full prepared, we could remove this flag boolean nereidsUseServerPrep = sessionVariable.enableServeSidePreparedStatement || mysqlCommand == MysqlCommand.COM_QUERY; if (nereidsUseServerPrep && sessionVariable.isEnableNereidsPlanner()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 3df4816d475f7f..002827a3cafa0d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -33,10 +33,9 @@ import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.plans.RelationId; -import org.apache.doris.nereids.trees.plans.commands.Command; +import org.apache.doris.nereids.trees.plans.PlaceholderId; import org.apache.doris.nereids.trees.plans.commands.ExecuteCommand; -import org.apache.doris.nereids.trees.plans.commands.PreparedCommand; +import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -149,9 +148,8 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) { auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); } - private void handleExecute(Command preparedCommand, long stmtId, PreparedStatementContext prepCtx) { - PreparedCommand prepareStmt = (PreparedCommand) preparedCommand; - int paramCount = prepareStmt.getParamLen(); + private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedStatementContext prepCtx) { + int paramCount = prepareCommand.getParamLen(); LOG.debug("execute prepared statement {}, paramCount {}", stmtId, paramCount); // null bitmap String stmtStr = ""; @@ -169,32 +167,32 @@ private void handleExecute(Command preparedCommand, long stmtId, PreparedStateme LOG.debug("code {}", typeCode); // assign type to placeholders typedPlaceholders.add( - prepareStmt.params().get(i).withNewMysqlColType(MysqlColType.fromCode(typeCode))); + prepareCommand.params().get(i).withNewMysqlColType(MysqlColType.fromCode(typeCode))); } // rewrite with new prepared statment with type info in placeholders - prepCtx.command = prepareStmt.withNewPreparedCommand(typedPlaceholders); - prepareStmt = (PreparedCommand) prepCtx.command; + prepCtx.command = prepareCommand.withPlaceholders(typedPlaceholders); + prepareCommand = (PrepareCommand) prepCtx.command; } // parse param data for (int i = 0; i < paramCount; ++i) { - RelationId exprId = prepareStmt.params().get(i).getExprId(); + PlaceholderId exprId = prepareCommand.params().get(i).getExprId(); if (isNull(nullbitmapData, i)) { statementContext.getIdToPlaceholderRealExpr().put(exprId, new org.apache.doris.nereids.trees.expressions.literal.NullLiteral()); continue; } - MysqlColType type = prepareStmt.params().get(i).getMysqlColType(); + MysqlColType type = prepareCommand.params().get(i).getMysqlColType(); Literal l = Literal.getLiteralByMysqlType(type, packetBuf); statementContext.getIdToPlaceholderRealExpr().put(exprId, l); } } - ExecuteCommand executeStmt = new ExecuteCommand(String.valueOf(stmtId), prepareStmt, statementContext); + ExecuteCommand executeStmt = new ExecuteCommand(String.valueOf(stmtId), prepareCommand, statementContext); // TODO set real origin statement if (LOG.isDebugEnabled()) { LOG.debug("executeStmt {}", executeStmt); } StatementBase stmt = new LogicalPlanAdapter(executeStmt, statementContext); - stmt.setOrigStmt(prepareStmt.getOriginalStmt()); + stmt.setOrigStmt(prepareCommand.getOriginalStmt()); executor = new StmtExecutor(ctx, stmt); ctx.setExecutor(executor); executor.execute(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java index d50a865bd70269..d54b2e5291c9cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java @@ -18,14 +18,10 @@ package org.apache.doris.qe; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.trees.plans.commands.Command; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; public class PreparedStatementContext { - private static final Logger LOG = LogManager.getLogger(PrepareStmtContext.class); - public Command command; + public PrepareCommand command; public ConnectContext ctx; StatementContext statementContext; public String stmtString; @@ -33,7 +29,7 @@ public class PreparedStatementContext { // Timestamp in millisecond last command starts at protected volatile long startTime; - public PreparedStatementContext(Command command, + public PreparedStatementContext(PrepareCommand command, ConnectContext ctx, StatementContext statementContext, String stmtString) { this.command = command; this.ctx = ctx; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 52be94fe49cf81..2a7421616f7b46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -152,7 +152,7 @@ import org.apache.doris.nereids.trees.plans.commands.DeleteFromUsingCommand; import org.apache.doris.nereids.trees.plans.commands.Forward; import org.apache.doris.nereids.trees.plans.commands.NotAllowFallback; -import org.apache.doris.nereids.trees.plans.commands.PreparedCommand; +import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; import org.apache.doris.nereids.trees.plans.commands.UpdateCommand; import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; @@ -685,8 +685,8 @@ private void executeByNereids(TUniqueId queryId) throws Exception { if (isForwardToMaster()) { throw new UserException("Forward master command is not supported for prepare statement"); } - logicalPlan = new PreparedCommand(String.valueOf(context.getStmtId()), - logicalPlan, statementContext.getParams(), originStmt); + logicalPlan = new PrepareCommand(String.valueOf(context.getStmtId()), + logicalPlan, statementContext.getPlaceholders(), originStmt); } // when we in transaction mode, we only support insert into command and transaction command From 0567d52d578b2f0d4ea119762b23d2ff2ded4fc0 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Fri, 31 May 2024 15:51:30 +0800 Subject: [PATCH 7/7] 2 --- .../rules/analysis/ExpressionAnalyzer.java | 4 ++-- .../nereids/trees/expressions/Placeholder.java | 16 ++++++++-------- .../expressions/visitor/ExpressionVisitor.java | 4 ++-- .../trees/plans/commands/ExecuteCommand.java | 6 +++--- .../trees/plans/commands/PrepareCommand.java | 8 ++++---- .../apache/doris/qe/MysqlConnectProcessor.java | 9 +++++---- 6 files changed, 24 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index e8f580e5fb0d98..0f4641cc425f6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -543,9 +543,9 @@ public Expression visitNot(Not not, ExpressionRewriteContext context) { } @Override - public Expression visitPlaceholder(Placeholder placeholderExpr, ExpressionRewriteContext context) { + public Expression visitPlaceholder(Placeholder placeholder, ExpressionRewriteContext context) { Expression realExpr = context.cascadesContext.getStatementContext() - .getIdToPlaceholderRealExpr().get(placeholderExpr.getExprId()); + .getIdToPlaceholderRealExpr().get(placeholder.getPlaceholderId()); return visit(realExpr, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java index 061360411d9864..9ae054d82a6598 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Placeholder.java @@ -31,21 +31,21 @@ * Placeholder for prepared statement */ public class Placeholder extends Expression implements LeafExpression { - private final PlaceholderId exprId; + private final PlaceholderId placeholderId; private final Optional mysqlColType; - public Placeholder(PlaceholderId exprId) { - this.exprId = exprId; + public Placeholder(PlaceholderId placeholderId) { + this.placeholderId = placeholderId; this.mysqlColType = Optional.empty(); } - public Placeholder(PlaceholderId exprId, MysqlColType mysqlColType) { - this.exprId = exprId; + public Placeholder(PlaceholderId placeholderId, MysqlColType mysqlColType) { + this.placeholderId = placeholderId; this.mysqlColType = Optional.of(mysqlColType); } - public PlaceholderId getExprId() { - return exprId; + public PlaceholderId getPlaceholderId() { + return placeholderId; } @Override @@ -69,7 +69,7 @@ public DataType getDataType() throws UnboundException { } public Placeholder withNewMysqlColType(MysqlColType mysqlColType) { - return new Placeholder(getExprId(), mysqlColType); + return new Placeholder(getPlaceholderId(), mysqlColType); } public MysqlColType getMysqlColType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index 7a7a60f9c37cca..294e96319e5c55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -506,8 +506,8 @@ public R visitMatchPhraseEdge(MatchPhraseEdge matchPhraseEdge, C context) { return visitMatch(matchPhraseEdge, context); } - public R visitPlaceholder(Placeholder placeholderExpr, C context) { - return visit(placeholderExpr, context); + public R visitPlaceholder(Placeholder placeholder, C context) { + return visit(placeholder, context); } public R visitAny(Any any, C context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java index 18925ae7756fbf..b098f883647b8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java @@ -62,7 +62,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { "prepare statement " + stmtName + " not found, maybe expired"); } PrepareCommand prepareCommand = (PrepareCommand) preparedStmtCtx.command; - LogicalPlanAdapter planAdapter = new LogicalPlanAdapter(prepareCommand.getInnerPlan(), executor.getContext() + LogicalPlanAdapter planAdapter = new LogicalPlanAdapter(prepareCommand.getLogicalPlan(), executor.getContext() .getStatementContext()); executor.setParsedStmt(planAdapter); // execute real statement @@ -74,8 +74,8 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { */ public String toSql() { // maybe slow - List realValueExpr = prepareCommand.params().stream() - .map(placeholder -> statementContext.getIdToPlaceholderRealExpr().get(placeholder.getExprId())) + List realValueExpr = prepareCommand.getPlaceholders().stream() + .map(placeholder -> statementContext.getIdToPlaceholderRealExpr().get(placeholder.getPlaceholderId())) .collect(Collectors.toList()); return "EXECUTE `" + stmtName + "`" + realValueExpr.stream().map(Expression::toSql).collect(Collectors.joining(", ", " USING ", "")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java index 8629ef4e4767ea..958fc4702836dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java @@ -66,15 +66,15 @@ public String getName() { return name; } - public List params() { + public List getPlaceholders() { return placeholders; } - public int getParamLen() { + public int placeholderCount() { return placeholders.size(); } - public LogicalPlan getInnerPlan() { + public LogicalPlan getLogicalPlan() { return logicalPlan; } @@ -88,7 +88,7 @@ public OriginStatement getOriginalStmt() { public List getLabels() { List labels = new ArrayList<>(); for (Placeholder parameter : placeholders) { - labels.add("$" + parameter.getExprId().asInt()); + labels.add("$" + parameter.getPlaceholderId().asInt()); } return labels; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 002827a3cafa0d..6c1507722d7b65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -149,7 +149,7 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) { } private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedStatementContext prepCtx) { - int paramCount = prepareCommand.getParamLen(); + int paramCount = prepareCommand.placeholderCount(); LOG.debug("execute prepared statement {}, paramCount {}", stmtId, paramCount); // null bitmap String stmtStr = ""; @@ -167,7 +167,8 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS LOG.debug("code {}", typeCode); // assign type to placeholders typedPlaceholders.add( - prepareCommand.params().get(i).withNewMysqlColType(MysqlColType.fromCode(typeCode))); + prepareCommand.getPlaceholders().get(i) + .withNewMysqlColType(MysqlColType.fromCode(typeCode))); } // rewrite with new prepared statment with type info in placeholders prepCtx.command = prepareCommand.withPlaceholders(typedPlaceholders); @@ -175,13 +176,13 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS } // parse param data for (int i = 0; i < paramCount; ++i) { - PlaceholderId exprId = prepareCommand.params().get(i).getExprId(); + PlaceholderId exprId = prepareCommand.getPlaceholders().get(i).getPlaceholderId(); if (isNull(nullbitmapData, i)) { statementContext.getIdToPlaceholderRealExpr().put(exprId, new org.apache.doris.nereids.trees.expressions.literal.NullLiteral()); continue; } - MysqlColType type = prepareCommand.params().get(i).getMysqlColType(); + MysqlColType type = prepareCommand.getPlaceholders().get(i).getMysqlColType(); Literal l = Literal.getLiteralByMysqlType(type, packetBuf); statementContext.getIdToPlaceholderRealExpr().put(exprId, l); }