From 6785d9a6ae0ab59d2bee64dae26bb509f32b705a Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 9 Jun 2020 11:45:21 +0800 Subject: [PATCH 1/6] order by w/o limit and offset in inline views, set operands and insert statements --- .../org/apache/doris/analysis/QueryStmt.java | 36 +++++++++---------- .../doris/planner/SingleNodePlanner.java | 4 ++- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java index 4da74e12d59160..2255fe93d1e7f5 100644 --- a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java @@ -285,27 +285,25 @@ protected void createSortInfo(Analyzer analyzer) throws AnalysisException { } sortInfo = new SortInfo(orderingExprs, isAscOrder, nullsFirstParams); - // order by w/o limit and offset in inline views, union operands and insert statements + // order by w/o limit and offset in inline views, set operands and insert statements // are ignored. - // TODO chenhao, open this when we don't limit rows subquery returns by SortNode. - /*if (!hasLimit() && !hasOffset() && !analyzer.isRootAnalyzer()) { - * evaluateOrderBy = false; - * // Return a warning that the order by was ignored. - * StringBuilder strBuilder = new StringBuilder(); - * strBuilder.append("Ignoring ORDER BY clause without LIMIT or OFFSET: "); - * strBuilder.append("ORDER BY "); - * strBuilder.append(orderByElements.get(0).toSql()); - * for (int i = 1; i < orderByElements.size(); ++i) { - * strBuilder.append(", ").append(orderByElements.get(i).toSql()); - * } - * strBuilder.append(".\nAn ORDER BY appearing in a view, subquery, union operand, "); - * strBuilder.append("or an insert/ctas statement has no effect on the query result "); - * strBuilder.append("unless a LIMIT and/or OFFSET is used in conjunction "); - * strBuilder.append("with the ORDER BY."); - * } else { - */ + if (!hasLimit() && !hasOffset() && !analyzer.isRootAnalyzer()) { + evaluateOrderBy = false; + // Return a warning that the order by was ignored. + StringBuilder strBuilder = new StringBuilder(); + strBuilder.append("Ignoring ORDER BY clause without LIMIT or OFFSET: "); + strBuilder.append("ORDER BY "); + strBuilder.append(orderByElements.get(0).toSql()); + for (int i = 1; i < orderByElements.size(); ++i) { + strBuilder.append(", ").append(orderByElements.get(i).toSql()); + } + strBuilder.append(".\nAn ORDER BY appearing in a view, subquery, union operand, "); + strBuilder.append("or an insert/ctas statement has no effect on the query result "); + strBuilder.append("unless a LIMIT and/or OFFSET is used in conjunction "); + strBuilder.append("with the ORDER BY."); + } else { evaluateOrderBy = true; - //} + } } /** diff --git a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 59f3a20777f971..f4b502be02cc10 100644 --- a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -2105,7 +2105,9 @@ private void putPredicatesOnFrom(SelectStmt stmt, Analyzer analyzer, List private List getBoundPredicates(Analyzer analyzer, TupleDescriptor tupleDesc) { final List tupleIds = Lists.newArrayList(); - tupleIds.add(tupleDesc.getId()); + if (tupleDesc != null) { + tupleIds.add(tupleDesc.getId()); + } return analyzer.getUnassignedConjuncts(tupleIds); } } From 62148664f85c77ff2d76546869d78d4ce2c533e3 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Tue, 9 Jun 2020 15:20:35 +0800 Subject: [PATCH 2/6] xx --- .../org/apache/doris/analysis/QueryStmt.java | 36 ++++++++++--------- .../org/apache/doris/qe/SessionVariable.java | 13 +++++++ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java index 2255fe93d1e7f5..a38891b78f4ea6 100644 --- a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java @@ -287,22 +287,26 @@ protected void createSortInfo(Analyzer analyzer) throws AnalysisException { sortInfo = new SortInfo(orderingExprs, isAscOrder, nullsFirstParams); // order by w/o limit and offset in inline views, set operands and insert statements // are ignored. - if (!hasLimit() && !hasOffset() && !analyzer.isRootAnalyzer()) { - evaluateOrderBy = false; - // Return a warning that the order by was ignored. - StringBuilder strBuilder = new StringBuilder(); - strBuilder.append("Ignoring ORDER BY clause without LIMIT or OFFSET: "); - strBuilder.append("ORDER BY "); - strBuilder.append(orderByElements.get(0).toSql()); - for (int i = 1; i < orderByElements.size(); ++i) { - strBuilder.append(", ").append(orderByElements.get(i).toSql()); - } - strBuilder.append(".\nAn ORDER BY appearing in a view, subquery, union operand, "); - strBuilder.append("or an insert/ctas statement has no effect on the query result "); - strBuilder.append("unless a LIMIT and/or OFFSET is used in conjunction "); - strBuilder.append("with the ORDER BY."); - } else { - evaluateOrderBy = true; + boolean allowIgnore = false; + if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isAllowIgnoreOrderBy()) { + allowIgnore = true; + } + if (allowIgnore && !hasLimit() && !hasOffset() && !analyzer.isRootAnalyzer()) { + evaluateOrderBy = false; + // Return a warning that the order by was ignored. + StringBuilder strBuilder = new StringBuilder(); + strBuilder.append("Ignoring ORDER BY clause without LIMIT or OFFSET: "); + strBuilder.append("ORDER BY "); + strBuilder.append(orderByElements.get(0).toSql()); + for (int i = 1; i < orderByElements.size(); ++i) { + strBuilder.append(", ").append(orderByElements.get(i).toSql()); + } + strBuilder.append(".\nAn ORDER BY appearing in a view, subquery, union operand, "); + strBuilder.append("or an insert/ctas statement has no effect on the query result "); + strBuilder.append("unless a LIMIT and/or OFFSET is used in conjunction "); + strBuilder.append("with the ORDER BY."); + } else { + evaluateOrderBy = true; } } diff --git a/fe/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/src/main/java/org/apache/doris/qe/SessionVariable.java index 393cdb02ca6c87..bf5dbf1f99c490 100644 --- a/fe/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -101,6 +101,8 @@ public class SessionVariable implements Serializable, Writable { public static final String MAX_SCAN_KEY_NUM = "max_scan_key_num"; public static final String MAX_PUSHDOWN_CONDITIONS_PER_COLUMN = "max_pushdown_conditions_per_column"; + public static final String ALLOW_IGNORE_ORDER_BY = "allow_ignore_order_by"; + // max memory used on every backend. @VariableMgr.VarAttr(name = EXEC_MEM_LIMIT) public long maxExecMemByte = 2147483648L; @@ -248,6 +250,9 @@ public class SessionVariable implements Serializable, Writable { private int maxScanKeyNum = -1; @VariableMgr.VarAttr(name = MAX_PUSHDOWN_CONDITIONS_PER_COLUMN) private int maxPushdownConditionsPerColumn = -1; + // TODO(Yangzhengguo) set this default value to true if we support spill data top disk + @VariableMgr.VarAttr(name = ALLOW_IGNORE_ORDER_BY) + private boolean allowIgnoreOrderBy = false; public long getMaxExecMemByte() { return maxExecMemByte; @@ -369,6 +374,10 @@ public int getCodegenLevel() { return codegenLevel; } + public boolean isAllowIgnoreOrderBy() { + return allowIgnoreOrderBy; + } + public void setMaxExecMemByte(long maxExecMemByte) { if (maxExecMemByte < MIN_EXEC_MEM_LIMIT) { this.maxExecMemByte = MIN_EXEC_MEM_LIMIT; @@ -474,6 +483,10 @@ public void setMaxPushdownConditionsPerColumn(int maxPushdownConditionsPerColumn this.maxPushdownConditionsPerColumn = maxPushdownConditionsPerColumn; } + public void setAllowIgnoreOrderBy(boolean allowIgnoreOrderBy) { + this.allowIgnoreOrderBy = allowIgnoreOrderBy; + } + // Serialize to thrift object // used for rest api public TQueryOptions toThrift() { From 5b39c348e071035b4a9eacff6ad14c5ef60ec8c5 Mon Sep 17 00:00:00 2001 From: yangzhg Date: Fri, 19 Jun 2020 09:52:28 +0800 Subject: [PATCH 3/6] add docs --- docs/en/administrator-guide/variables.md | 4 ++++ docs/zh-CN/administrator-guide/variables.md | 3 +++ 2 files changed, 7 insertions(+) diff --git a/docs/en/administrator-guide/variables.md b/docs/en/administrator-guide/variables.md index c5da687f325d14..350c2d6b7f0333 100644 --- a/docs/en/administrator-guide/variables.md +++ b/docs/en/administrator-guide/variables.md @@ -338,3 +338,7 @@ SET forward_to_master = concat('tr', 'u', 'e'); * `rewrite_count_distinct_to_bitmap_hll` Whether to rewrite count distinct queries of bitmap and HLL types as bitmap_union_count and hll_union_agg. + +* `allow_ignore_order_by` + Used to ignore order by w/o limit and offset in inline views, set operands and insert statements. + diff --git a/docs/zh-CN/administrator-guide/variables.md b/docs/zh-CN/administrator-guide/variables.md index c592fc8c96292e..b4a4626eecedf3 100644 --- a/docs/zh-CN/administrator-guide/variables.md +++ b/docs/zh-CN/administrator-guide/variables.md @@ -338,3 +338,6 @@ SET forward_to_master = concat('tr', 'u', 'e'); 是否将 bitmap 和 hll 类型的 count distinct 查询重写为 bitmap_union_count 和 hll_union_agg 。 +* `allow_ignore_order_by` + + 用来忽略没有limit和offset语句的集合操作、inline view、insert 中的order by 语句，提升查询性能。 From 25cbe16f7a275234bd03caf26cdcce213f4a16da Mon Sep 17 00:00:00 2001 From: yangzhg Date: Wed, 24 Jun 2020 11:39:03 +0800 Subject: [PATCH 4/6] remvoe session variable --- docs/en/administrator-guide/variables.md | 4 ---- docs/zh-CN/administrator-guide/variables.md | 3 --- .../java/org/apache/doris/analysis/QueryStmt.java | 6 +----- .../java/org/apache/doris/qe/SessionVariable.java | 13 ------------- 4 files changed, 1 insertion(+), 25 deletions(-) diff --git a/docs/en/administrator-guide/variables.md b/docs/en/administrator-guide/variables.md index 350c2d6b7f0333..c5da687f325d14 100644 --- a/docs/en/administrator-guide/variables.md +++ b/docs/en/administrator-guide/variables.md @@ -338,7 +338,3 @@ SET forward_to_master = concat('tr', 'u', 'e'); * `rewrite_count_distinct_to_bitmap_hll` Whether to rewrite count distinct queries of bitmap and HLL types as bitmap_union_count and hll_union_agg. - -* `allow_ignore_order_by` - Used to ignore order by w/o limit and offset in inline views, set operands and insert statements. - diff --git a/docs/zh-CN/administrator-guide/variables.md b/docs/zh-CN/administrator-guide/variables.md index b4a4626eecedf3..c592fc8c96292e 100644 --- a/docs/zh-CN/administrator-guide/variables.md +++ b/docs/zh-CN/administrator-guide/variables.md @@ -338,6 +338,3 @@ SET forward_to_master = concat('tr', 'u', 'e'); 是否将 bitmap 和 hll 类型的 count distinct 查询重写为 bitmap_union_count 和 hll_union_agg 。 -* `allow_ignore_order_by` - - 用来忽略没有limit和offset语句的集合操作、inline view、insert 中的order by 语句，提升查询性能。 diff --git a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java index a38891b78f4ea6..b9d974f87b9e5c 100644 --- a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java @@ -287,11 +287,7 @@ protected void createSortInfo(Analyzer analyzer) throws AnalysisException { sortInfo = new SortInfo(orderingExprs, isAscOrder, nullsFirstParams); // order by w/o limit and offset in inline views, set operands and insert statements // are ignored. - boolean allowIgnore = false; - if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isAllowIgnoreOrderBy()) { - allowIgnore = true; - } - if (allowIgnore && !hasLimit() && !hasOffset() && !analyzer.isRootAnalyzer()) { + if (!hasLimit() && !hasOffset() && !analyzer.isRootAnalyzer()) { evaluateOrderBy = false; // Return a warning that the order by was ignored. StringBuilder strBuilder = new StringBuilder(); diff --git a/fe/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/src/main/java/org/apache/doris/qe/SessionVariable.java index bf5dbf1f99c490..393cdb02ca6c87 100644 --- a/fe/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -101,8 +101,6 @@ public class SessionVariable implements Serializable, Writable { public static final String MAX_SCAN_KEY_NUM = "max_scan_key_num"; public static final String MAX_PUSHDOWN_CONDITIONS_PER_COLUMN = "max_pushdown_conditions_per_column"; - public static final String ALLOW_IGNORE_ORDER_BY = "allow_ignore_order_by"; - // max memory used on every backend. @VariableMgr.VarAttr(name = EXEC_MEM_LIMIT) public long maxExecMemByte = 2147483648L; @@ -250,9 +248,6 @@ public class SessionVariable implements Serializable, Writable { private int maxScanKeyNum = -1; @VariableMgr.VarAttr(name = MAX_PUSHDOWN_CONDITIONS_PER_COLUMN) private int maxPushdownConditionsPerColumn = -1; - // TODO(Yangzhengguo) set this default value to true if we support spill data top disk - @VariableMgr.VarAttr(name = ALLOW_IGNORE_ORDER_BY) - private boolean allowIgnoreOrderBy = false; public long getMaxExecMemByte() { return maxExecMemByte; @@ -374,10 +369,6 @@ public int getCodegenLevel() { return codegenLevel; } - public boolean isAllowIgnoreOrderBy() { - return allowIgnoreOrderBy; - } - public void setMaxExecMemByte(long maxExecMemByte) { if (maxExecMemByte < MIN_EXEC_MEM_LIMIT) { this.maxExecMemByte = MIN_EXEC_MEM_LIMIT; @@ -483,10 +474,6 @@ public void setMaxPushdownConditionsPerColumn(int maxPushdownConditionsPerColumn this.maxPushdownConditionsPerColumn = maxPushdownConditionsPerColumn; } - public void setAllowIgnoreOrderBy(boolean allowIgnoreOrderBy) { - this.allowIgnoreOrderBy = allowIgnoreOrderBy; - } - // Serialize to thrift object // used for rest api public TQueryOptions toThrift() { From f2a509a54bb0b437610bceb3965af0b5d961d55e Mon Sep 17 00:00:00 2001 From: yangzhg Date: Wed, 1 Jul 2020 15:42:16 +0800 Subject: [PATCH 5/6] log warning --- fe/src/main/java/org/apache/doris/analysis/QueryStmt.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java index b9d974f87b9e5c..bf6b53a171fc98 100644 --- a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java @@ -301,6 +301,7 @@ protected void createSortInfo(Analyzer analyzer) throws AnalysisException { strBuilder.append("or an insert/ctas statement has no effect on the query result "); strBuilder.append("unless a LIMIT and/or OFFSET is used in conjunction "); strBuilder.append("with the ORDER BY."); + LOG.warn(strBuilder.toString()); } else { evaluateOrderBy = true; } From 4dc5db4b67bbfcd5aeba87b2b76c74b6f9bd8bdc Mon Sep 17 00:00:00 2001 From: yangzhg Date: Wed, 1 Jul 2020 16:12:02 +0800 Subject: [PATCH 6/6] add_log --- fe/src/main/java/org/apache/doris/analysis/QueryStmt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java index bf6b53a171fc98..32cce4524784b2 100644 --- a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java @@ -301,7 +301,7 @@ protected void createSortInfo(Analyzer analyzer) throws AnalysisException { strBuilder.append("or an insert/ctas statement has no effect on the query result "); strBuilder.append("unless a LIMIT and/or OFFSET is used in conjunction "); strBuilder.append("with the ORDER BY."); - LOG.warn(strBuilder.toString()); + LOG.info(strBuilder.toString()); } else { evaluateOrderBy = true; }