Skip to content

Add PreAgg Hint #1617

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 2 commits into from
Sep 6, 2019
Merged

Add PreAgg Hint #1617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fb8eef0
plan hint support CommentedHint style.
worker24h Sep 3, 2019
ace5331
Add PreAgg Hint
worker24h Aug 1, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions fe/src/main/cup/sql_parser.cup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_A
terminal COMMA, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, DIVIDE, MOD, ADD, SUBTRACT;
terminal BITAND, BITOR, BITXOR, BITNOT;
terminal EQUAL, NOT, LESSTHAN, GREATERTHAN, SET_VAR;
terminal COMMENTED_PLAN_HINT_START, COMMENTED_PLAN_HINT_END;
terminal String IDENT;
terminal String NUMERIC_OVERFLOW;
terminal Long INTEGER_LITERAL;
Expand Down Expand Up @@ -338,6 +339,7 @@ nonterminal ArrayList<String> opt_plan_hints;
nonterminal ArrayList<String> opt_sort_hints;
nonterminal Expr sign_chain_expr;
nonterminal Qualifier union_op;
nonterminal ArrayList<String> opt_common_hints;

nonterminal ArrayList<PartitionName> opt_partition_name_list, partition_name_list;
nonterminal PartitionName partition_name;
Expand Down Expand Up @@ -2973,9 +2975,24 @@ base_table_ref_list ::=
;

base_table_ref ::=
table_name:name opt_using_partition:parts opt_table_alias:alias
table_name:name opt_using_partition:parts opt_table_alias:alias opt_common_hints:common_hints
{:
RESULT = new TableRef(name, alias, parts);
RESULT = new TableRef(name, alias, parts, common_hints);
:}
;

opt_common_hints ::=
COMMENTED_PLAN_HINT_START ident_list:l COMMENTED_PLAN_HINT_END
{:
RESULT = l;
:}
| LBRACKET ident_list:l RBRACKET
{:
RESULT = l;
:}
|
{:
RESULT = null;
:}
;

Expand Down Expand Up @@ -3055,6 +3072,10 @@ opt_plan_hints ::=
}
RESULT = hints;
:}
| COMMENTED_PLAN_HINT_START ident_list:l COMMENTED_PLAN_HINT_END
{:
RESULT = l;
:}
| LBRACKET ident_list:l RBRACKET
{:
RESULT = l;
Expand Down
1 change: 1 addition & 0 deletions fe/src/main/java/org/apache/doris/analysis/BaseTableRef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
isAnalyzed = true; // true that we have assigned desc
analyzeJoin(analyzer);
analyzeSortHints();
analyzeHints();
}
}

16 changes: 10 additions & 6 deletions fe/src/main/java/org/apache/doris/analysis/InlineViewRef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@

package org.apache.doris.analysis;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.InlineView;
import org.apache.doris.catalog.View;
Expand All @@ -25,12 +28,8 @@
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.rewrite.ExprRewriter;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -265,7 +264,12 @@ public TupleDescriptor createTupleDescriptor(Analyzer analyzer) throws AnalysisE
}

columnSet.add(colAlias);
columnList.add(new Column(colAlias, selectItemExpr.getType().getPrimitiveType()));
if (selectItemExpr instanceof SlotRef && ((SlotRef)selectItemExpr).getDesc().getColumn() != null) {
SlotRef slotRef = (SlotRef) selectItemExpr;
columnList.add(new Column(slotRef.getDesc().getColumn()));
} else {
columnList.add(new Column(colAlias, selectItemExpr.getType().getPrimitiveType()));
}
}
InlineView inlineView = (view != null) ? new InlineView(view, columnList) : new InlineView(getExplicitAlias(), columnList);

Expand Down
40 changes: 32 additions & 8 deletions fe/src/main/java/org/apache/doris/analysis/TableRef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@

package org.apache.doris.analysis;

import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
Expand All @@ -25,12 +29,6 @@
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.rewrite.ExprRewriter;

import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

Expand Down Expand Up @@ -90,7 +88,8 @@ public class TableRef implements ParseNode, Writable {
protected List<String> usingColNames;
private ArrayList<String> joinHints;
private ArrayList<String> sortHints;

private ArrayList<String> commonHints; //The Hints is set by user
private boolean isForcePreAggOpened;
// ///////////////////////////////////////
// BEGIN: Members that need to be reset()

Expand Down Expand Up @@ -137,6 +136,10 @@ public TableRef(TableName name, String alias) {
}

public TableRef(TableName name, String alias, List<String> partitions) {
this(name, alias, partitions, null);
}

public TableRef(TableName name, String alias, List<String> partitions, ArrayList<String> commonHints) {
this.name = name;
if (alias != null) {
aliases_ = new String[] { alias };
Expand All @@ -145,9 +148,9 @@ public TableRef(TableName name, String alias, List<String> partitions) {
hasExplicitAlias_ = false;
}
this.partitions = partitions;
this.commonHints = commonHints;
isAnalyzed = false;
}

// Only used to clone
// this will reset all the 'analyzed' stuff
protected TableRef(TableRef other) {
Expand All @@ -163,6 +166,7 @@ protected TableRef(TableRef other) {
onClause = (other.onClause != null) ? other.onClause.clone().reset() : null;
partitions =
(other.partitions != null) ? Lists.newArrayList(other.partitions) : null;
commonHints = other.commonHints;

usingColNames =
(other.usingColNames != null) ? Lists.newArrayList(other.usingColNames) : null;
Expand Down Expand Up @@ -301,6 +305,10 @@ public boolean isPartitionJoin() {
return isPartitionJoin;
}

public boolean isForcePreAggOpened() {
return isForcePreAggOpened;
}

public void setSortHints(ArrayList<String> hints) {
this.sortHints = hints;
}
Expand Down Expand Up @@ -342,6 +350,22 @@ private void analyzeJoinHints() throws AnalysisException {
}
}

/**
* Parse PreAgg hints.
*/
protected void analyzeHints() throws AnalysisException {
if (commonHints == null || commonHints.isEmpty()) {
return;
}
// Currently only 'PREAGGOPEN' is supported
for (String hint : commonHints) {
if (hint.toUpperCase().equals("PREAGGOPEN")) {
isForcePreAggOpened = true;
break;
}
}
}

/**
* Analyze the join clause.
* The join clause can only be analyzed after the left table has been analyzed
Expand Down
26 changes: 16 additions & 10 deletions fe/src/main/java/org/apache/doris/planner/OlapScanNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@

package org.apache.doris.planner;

import com.google.common.base.Objects;
import com.google.common.base.Objects.ToStringHelper;
import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.BaseTableRef;
import org.apache.doris.analysis.TupleDescriptor;
Expand Down Expand Up @@ -50,15 +57,6 @@
import org.apache.doris.thrift.TScanRange;
import org.apache.doris.thrift.TScanRangeLocation;
import org.apache.doris.thrift.TScanRangeLocations;

import com.google.common.base.Objects;
import com.google.common.base.Objects.ToStringHelper;
import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

Expand All @@ -80,6 +78,7 @@ public class OlapScanNode extends ScanNode {
private boolean isPreAggregation = false;
private String reasonOfPreAggregation = null;
private boolean canTurnOnPreAggr = true;
private boolean forceOpenPreAgg = false;
private ArrayList<String> tupleColumns = new ArrayList<String>();
private HashSet<String> predicateColumns = new HashSet<String>();
private OlapTable olapTable = null;
Expand Down Expand Up @@ -109,7 +108,6 @@ public void setIsPreAggregation(boolean isPreAggregation, String reason) {
this.reasonOfPreAggregation = reason;
}


public boolean isPreAggregation() {
return isPreAggregation;
}
Expand All @@ -122,6 +120,14 @@ public void setCanTurnOnPreAggr(boolean canChangePreAggr) {
this.canTurnOnPreAggr = canChangePreAggr;
}

public boolean getForceOpenPreAgg() {
return forceOpenPreAgg;
}

public void setForceOpenPreAgg(boolean forceOpenPreAgg) {
this.forceOpenPreAgg = forceOpenPreAgg;
}

public OlapTable getOlapTable() {
return olapTable;
}
Expand Down
35 changes: 20 additions & 15 deletions fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@

package org.apache.doris.planner;

import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.doris.analysis.AggregateInfo;
import org.apache.doris.analysis.AnalyticInfo;
import org.apache.doris.analysis.Analyzer;
Expand Down Expand Up @@ -52,13 +57,6 @@
import org.apache.doris.common.Pair;
import org.apache.doris.common.Reference;
import org.apache.doris.common.UserException;

import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

Expand Down Expand Up @@ -317,13 +315,18 @@ private PlanNode addUnassignedConjuncts(
private void turnOffPreAgg(AggregateInfo aggInfo, SelectStmt selectStmt, Analyzer analyzer, PlanNode root) {
String turnOffReason = null;
do {
if (null == aggInfo) {
turnOffReason = "No AggregateInfo";
if (!(root instanceof OlapScanNode)) {
turnOffReason = "left-deep Node is not OlapScanNode";
break;
}

if (!(root instanceof OlapScanNode)) {
turnOffReason = "left-deep Node is not OlapScanNode";
if (((OlapScanNode)root).getForceOpenPreAgg()) {
((OlapScanNode)root).setIsPreAggregation(true, "");
return;
}

if (null == aggInfo) {
turnOffReason = "No AggregateInfo";
break;
}

Expand Down Expand Up @@ -394,8 +397,8 @@ private void turnOffPreAgg(AggregateInfo aggInfo, SelectStmt selectStmt, Analyze
for (SlotDescriptor slot : selectStmt.getTableRefs().get(0).getDesc().getSlots()) {
if (!slot.getColumn().isKey()) {
if (conjunctSlotIds.contains(slot.getId())) {
turnOffReason = "conjunct on " + slot.getColumn().getName() +
" which is StorageEngine value column";
turnOffReason = "conjunct on `" + slot.getColumn().getName() +
"` which is StorageEngine value column";
valueColumnValidate = false;
break;
}
Expand Down Expand Up @@ -630,7 +633,7 @@ private PlanNode createSelectPlan(SelectStmt selectStmt, Analyzer analyzer, long
rowTuples.addAll(tblRef.getMaterializedTupleIds());
}

if (analyzer.hasEmptySpjResultSet()) {
if (analyzer.hasEmptySpjResultSet()) {
final PlanNode emptySetNode = new EmptySetNode(ctx_.getNextNodeId(), rowTuples);
emptySetNode.init(analyzer);
emptySetNode.setOutputSmap(selectStmt.getBaseTblSmap());
Expand Down Expand Up @@ -1143,7 +1146,9 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef)

switch (tblRef.getTable().getType()) {
case OLAP:
scanNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode");
OlapScanNode olapNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode");
olapNode.setForceOpenPreAgg(tblRef.isForcePreAggOpened());
scanNode = olapNode;
break;
case MYSQL:
scanNode = new MysqlScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), (MysqlTable) tblRef.getTable());
Expand Down
Loading