diff --git a/fe/src/main/cup/sql_parser.cup b/fe/src/main/cup/sql_parser.cup index 5dbcfe99d66977..7ac89fd1b9ec5a 100644 --- a/fe/src/main/cup/sql_parser.cup +++ b/fe/src/main/cup/sql_parser.cup @@ -231,6 +231,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_A terminal COMMA, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, DIVIDE, MOD, ADD, SUBTRACT; terminal BITAND, BITOR, BITXOR, BITNOT; terminal EQUAL, NOT, LESSTHAN, GREATERTHAN, SET_VAR; +terminal COMMENTED_PLAN_HINT_START, COMMENTED_PLAN_HINT_END; terminal String IDENT; terminal String NUMERIC_OVERFLOW; terminal Long INTEGER_LITERAL; @@ -338,6 +339,7 @@ nonterminal ArrayList opt_plan_hints; nonterminal ArrayList opt_sort_hints; nonterminal Expr sign_chain_expr; nonterminal Qualifier union_op; +nonterminal ArrayList opt_common_hints; nonterminal ArrayList opt_partition_name_list, partition_name_list; nonterminal PartitionName partition_name; @@ -2973,9 +2975,24 @@ base_table_ref_list ::= ; base_table_ref ::= - table_name:name opt_using_partition:parts opt_table_alias:alias + table_name:name opt_using_partition:parts opt_table_alias:alias opt_common_hints:common_hints {: - RESULT = new TableRef(name, alias, parts); + RESULT = new TableRef(name, alias, parts, common_hints); + :} + ; + +opt_common_hints ::= + COMMENTED_PLAN_HINT_START ident_list:l COMMENTED_PLAN_HINT_END + {: + RESULT = l; + :} + | LBRACKET ident_list:l RBRACKET + {: + RESULT = l; + :} + | + {: + RESULT = null; :} ; @@ -3055,6 +3072,10 @@ opt_plan_hints ::= } RESULT = hints; :} + | COMMENTED_PLAN_HINT_START ident_list:l COMMENTED_PLAN_HINT_END + {: + RESULT = l; + :} | LBRACKET ident_list:l RBRACKET {: RESULT = l; diff --git a/fe/src/main/java/org/apache/doris/analysis/BaseTableRef.java b/fe/src/main/java/org/apache/doris/analysis/BaseTableRef.java index cc6d38f96f04e0..3eceea0fbb3949 100644 --- a/fe/src/main/java/org/apache/doris/analysis/BaseTableRef.java +++ b/fe/src/main/java/org/apache/doris/analysis/BaseTableRef.java @@ -70,6 +70,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException { isAnalyzed = true; // true that we have assigned desc analyzeJoin(analyzer); analyzeSortHints(); + analyzeHints(); } } diff --git a/fe/src/main/java/org/apache/doris/analysis/InlineViewRef.java b/fe/src/main/java/org/apache/doris/analysis/InlineViewRef.java index f92df8608d5df1..973e4bbd44980b 100644 --- a/fe/src/main/java/org/apache/doris/analysis/InlineViewRef.java +++ b/fe/src/main/java/org/apache/doris/analysis/InlineViewRef.java @@ -17,6 +17,9 @@ package org.apache.doris.analysis; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.InlineView; import org.apache.doris.catalog.View; @@ -25,12 +28,8 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; import org.apache.doris.rewrite.ExprRewriter; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.List; @@ -265,7 +264,12 @@ public TupleDescriptor createTupleDescriptor(Analyzer analyzer) throws AnalysisE } columnSet.add(colAlias); - columnList.add(new Column(colAlias, selectItemExpr.getType().getPrimitiveType())); + if (selectItemExpr instanceof SlotRef && ((SlotRef)selectItemExpr).getDesc().getColumn() != null) { + SlotRef slotRef = (SlotRef) selectItemExpr; + columnList.add(new Column(slotRef.getDesc().getColumn())); + } else { + columnList.add(new Column(colAlias, selectItemExpr.getType().getPrimitiveType())); + } } InlineView inlineView = (view != null) ? new InlineView(view, columnList) : new InlineView(getExplicitAlias(), columnList); diff --git a/fe/src/main/java/org/apache/doris/analysis/TableRef.java b/fe/src/main/java/org/apache/doris/analysis/TableRef.java index c997d1d80666a3..52e11d09cde914 100644 --- a/fe/src/main/java/org/apache/doris/analysis/TableRef.java +++ b/fe/src/main/java/org/apache/doris/analysis/TableRef.java @@ -17,6 +17,10 @@ package org.apache.doris.analysis; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; @@ -25,12 +29,6 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.rewrite.ExprRewriter; - -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -90,7 +88,8 @@ public class TableRef implements ParseNode, Writable { protected List usingColNames; private ArrayList joinHints; private ArrayList sortHints; - + private ArrayList commonHints; //The Hints is set by user + private boolean isForcePreAggOpened; // /////////////////////////////////////// // BEGIN: Members that need to be reset() @@ -137,6 +136,10 @@ public TableRef(TableName name, String alias) { } public TableRef(TableName name, String alias, List partitions) { + this(name, alias, partitions, null); + } + + public TableRef(TableName name, String alias, List partitions, ArrayList commonHints) { this.name = name; if (alias != null) { aliases_ = new String[] { alias }; @@ -145,9 +148,9 @@ public TableRef(TableName name, String alias, List partitions) { hasExplicitAlias_ = false; } this.partitions = partitions; + this.commonHints = commonHints; isAnalyzed = false; } - // Only used to clone // this will reset all the 'analyzed' stuff protected TableRef(TableRef other) { @@ -163,6 +166,7 @@ protected TableRef(TableRef other) { onClause = (other.onClause != null) ? other.onClause.clone().reset() : null; partitions = (other.partitions != null) ? Lists.newArrayList(other.partitions) : null; + commonHints = other.commonHints; usingColNames = (other.usingColNames != null) ? Lists.newArrayList(other.usingColNames) : null; @@ -301,6 +305,10 @@ public boolean isPartitionJoin() { return isPartitionJoin; } + public boolean isForcePreAggOpened() { + return isForcePreAggOpened; + } + public void setSortHints(ArrayList hints) { this.sortHints = hints; } @@ -342,6 +350,22 @@ private void analyzeJoinHints() throws AnalysisException { } } + /** + * Parse PreAgg hints. + */ + protected void analyzeHints() throws AnalysisException { + if (commonHints == null || commonHints.isEmpty()) { + return; + } + // Currently only 'PREAGGOPEN' is supported + for (String hint : commonHints) { + if (hint.toUpperCase().equals("PREAGGOPEN")) { + isForcePreAggOpened = true; + break; + } + } + } + /** * Analyze the join clause. * The join clause can only be analyzed after the left table has been analyzed diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index 669a58d67708c0..a791da2f2525f8 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -17,6 +17,13 @@ package org.apache.doris.planner; +import com.google.common.base.Objects; +import com.google.common.base.Objects.ToStringHelper; +import com.google.common.base.Preconditions; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Range; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.BaseTableRef; import org.apache.doris.analysis.TupleDescriptor; @@ -50,15 +57,6 @@ import org.apache.doris.thrift.TScanRange; import org.apache.doris.thrift.TScanRangeLocation; import org.apache.doris.thrift.TScanRangeLocations; - -import com.google.common.base.Objects; -import com.google.common.base.Objects.ToStringHelper; -import com.google.common.base.Preconditions; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Range; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -80,6 +78,7 @@ public class OlapScanNode extends ScanNode { private boolean isPreAggregation = false; private String reasonOfPreAggregation = null; private boolean canTurnOnPreAggr = true; + private boolean forceOpenPreAgg = false; private ArrayList tupleColumns = new ArrayList(); private HashSet predicateColumns = new HashSet(); private OlapTable olapTable = null; @@ -109,7 +108,6 @@ public void setIsPreAggregation(boolean isPreAggregation, String reason) { this.reasonOfPreAggregation = reason; } - public boolean isPreAggregation() { return isPreAggregation; } @@ -122,6 +120,14 @@ public void setCanTurnOnPreAggr(boolean canChangePreAggr) { this.canTurnOnPreAggr = canChangePreAggr; } + public boolean getForceOpenPreAgg() { + return forceOpenPreAgg; + } + + public void setForceOpenPreAgg(boolean forceOpenPreAgg) { + this.forceOpenPreAgg = forceOpenPreAgg; + } + public OlapTable getOlapTable() { return olapTable; } diff --git a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index e4b6269ec744d3..145a814e1baa91 100644 --- a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -17,6 +17,11 @@ package org.apache.doris.planner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.doris.analysis.AggregateInfo; import org.apache.doris.analysis.AnalyticInfo; import org.apache.doris.analysis.Analyzer; @@ -52,13 +57,6 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.Reference; import org.apache.doris.common.UserException; - -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -317,13 +315,18 @@ private PlanNode addUnassignedConjuncts( private void turnOffPreAgg(AggregateInfo aggInfo, SelectStmt selectStmt, Analyzer analyzer, PlanNode root) { String turnOffReason = null; do { - if (null == aggInfo) { - turnOffReason = "No AggregateInfo"; + if (!(root instanceof OlapScanNode)) { + turnOffReason = "left-deep Node is not OlapScanNode"; break; } - if (!(root instanceof OlapScanNode)) { - turnOffReason = "left-deep Node is not OlapScanNode"; + if (((OlapScanNode)root).getForceOpenPreAgg()) { + ((OlapScanNode)root).setIsPreAggregation(true, ""); + return; + } + + if (null == aggInfo) { + turnOffReason = "No AggregateInfo"; break; } @@ -394,8 +397,8 @@ private void turnOffPreAgg(AggregateInfo aggInfo, SelectStmt selectStmt, Analyze for (SlotDescriptor slot : selectStmt.getTableRefs().get(0).getDesc().getSlots()) { if (!slot.getColumn().isKey()) { if (conjunctSlotIds.contains(slot.getId())) { - turnOffReason = "conjunct on " + slot.getColumn().getName() + - " which is StorageEngine value column"; + turnOffReason = "conjunct on `" + slot.getColumn().getName() + + "` which is StorageEngine value column"; valueColumnValidate = false; break; } @@ -630,7 +633,7 @@ private PlanNode createSelectPlan(SelectStmt selectStmt, Analyzer analyzer, long rowTuples.addAll(tblRef.getMaterializedTupleIds()); } - if (analyzer.hasEmptySpjResultSet()) { + if (analyzer.hasEmptySpjResultSet()) { final PlanNode emptySetNode = new EmptySetNode(ctx_.getNextNodeId(), rowTuples); emptySetNode.init(analyzer); emptySetNode.setOutputSmap(selectStmt.getBaseTblSmap()); @@ -1143,7 +1146,9 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef) switch (tblRef.getTable().getType()) { case OLAP: - scanNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode"); + OlapScanNode olapNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode"); + olapNode.setForceOpenPreAgg(tblRef.isForcePreAggOpened()); + scanNode = olapNode; break; case MYSQL: scanNode = new MysqlScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), (MysqlTable) tblRef.getTable()); diff --git a/fe/src/main/jflex/sql_scanner.flex b/fe/src/main/jflex/sql_scanner.flex index 07d1fb61999444..b91d1f189f0b76 100644 --- a/fe/src/main/jflex/sql_scanner.flex +++ b/fe/src/main/jflex/sql_scanner.flex @@ -439,20 +439,36 @@ QuotedIdentifier = \`(\`\`|[^\`])*\` SingleQuoteStringLiteral = \'(\\.|[^\\\'])*\' DoubleQuoteStringLiteral = \"(\\.|[^\\\"])*\" -// Both types of plan hints must appear within a single line. -TraditionalCommentedPlanHints = "/*" [ ]* "+" [^\r\n*]* "*/" -// Must end with a line terminator. -EndOfLineCommentedPlanHints = "--" [ ]* "+" {NonTerminator}* {LineTerminator} - FLit1 = [0-9]+ \. [0-9]* FLit2 = \. [0-9]+ FLit3 = [0-9]+ Exponent = [eE] [+-]? [0-9]+ DoubleLiteral = ({FLit1}|{FLit2}|{FLit3}) {Exponent}? +EolHintBegin = "--" " "* "+" +CommentedHintBegin = "/*" " "* "+" +CommentedHintEnd = "*/" + +// Both types of plan hints must appear within a single line. +HintContent = " "* "+" [^\r\n]* + Comment = {TraditionalComment} | {EndOfLineComment} -TraditionalComment = "/*" [^*] ~"*/" | "/*" "*"+ "/" -EndOfLineComment = "--" {NonTerminator}* {LineTerminator}? + +// Match anything that has a comment end (*/) in it. +ContainsCommentEnd = [^]* "*/" [^]* +// Match anything that has a line terminator in it. +ContainsLineTerminator = [^]* {LineTerminator} [^]* + +// A traditional comment is anything that starts and ends like a comment and has neither a +// plan hint inside nor a CommentEnd (*/). +TraditionalComment = "/*" !({HintContent}|{ContainsCommentEnd}) "*/" +// Similar for a end-of-line comment. +EndOfLineComment = "--" !({HintContent}|{ContainsLineTerminator}) {LineTerminator}? + +// This additional state is needed because newlines signal the end of a end-of-line hint +// if one has been started earlier. Hence we need to discern between newlines within and +// outside of end-of-line hints. +%state EOLHINT %% @@ -512,6 +528,24 @@ EndOfLineComment = "--" {NonTerminator}* {LineTerminator}? escapeBackSlash(yytext().substring(1, yytext().length()-1))); } +{CommentedHintBegin} { + return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_START, null); +} + +{CommentedHintEnd} { + return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_END, null); +} + +{EolHintBegin} { + yybegin(EOLHINT); + return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_START, null); +} + + {LineTerminator} { + yybegin(YYINITIAL); + return newToken(SqlParserSymbols.COMMENTED_PLAN_HINT_END, null); +} + {IntegerLiteral} { BigInteger val = null; try { @@ -542,19 +576,5 @@ EndOfLineComment = "--" {NonTerminator}* {LineTerminator}? return newToken(SqlParserSymbols.DECIMAL_LITERAL, decimal_val); } -{TraditionalCommentedPlanHints} { - String text = yytext(); - // Remove everything before the first '+' as well as the trailing "*/" - String hintStr = text.substring(text.indexOf('+') + 1, text.length() - 2); - return newToken(SqlParserSymbols.COMMENTED_PLAN_HINTS, hintStr.trim()); -} - -{EndOfLineCommentedPlanHints} { - String text = yytext(); - // Remove everything before the first '+' - String hintStr = text.substring(text.indexOf('+') + 1); - return newToken(SqlParserSymbols.COMMENTED_PLAN_HINTS, hintStr.trim()); -} - {Comment} { /* ignore */ } {Whitespace} { /* ignore */ }