From c3434ea88d80fe876416955dbc1c18065269e70c Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 6 Nov 2024 14:53:33 +0800 Subject: [PATCH 1/3] fix multi column partition prune --- .../rules/OneRangePartitionEvaluator.java | 1 + .../test_date_trunc_prune.groovy | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index c3345c346932a1..fa0bd3d6781d2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -174,6 +174,7 @@ public List> getOnePartitionInputs() { @Override public Expression evaluate(Expression expression, Map currentInputs) { + rangeMap.clear(); Map defaultColumnRanges = currentInputs.values().iterator().next().columnRanges; rangeMap.putAll(defaultColumnRanges); EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs)); diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy index 4ad3afbb9db040..d6f40e39a8df68 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy @@ -334,4 +334,29 @@ suite("test_date_trunc_prune") { ( NOT ( ( table1 . `col_int_undef_signed` != table1 . `col_int_undef_signed` ) AND table1 . `col_date_undef_signed` <= '2025-06-18' ) AND table1 . `col_date_undef_signed` IN ( '2023-12-20' ) ) GROUP BY field1 ORDER BY field1 LIMIT 1000; """ + + // test multi column partition table and predicate with date_trunc + sql "drop table if exists t_multi_column_partition" + sql """ + create table t_multi_column_partition(a int, dt datetime, v int) partition by range(a, dt) + ( + partition p0 values [(0,'2024-01-01 00:00:00'), (10,'2024-01-10 00:00:00')), + partition p10 values [(10,'2024-01-10 00:00:00'), (20,'2024-01-20 00:00:00')), + partition p20 values [(20,'2024-01-20 00:00:00'), (30,'2024-01-31 00:00:00')), + partition p30 values [(30,'2024-01-31 00:00:00'), (40,'2024-02-10 00:00:00')), + partition p40 values [(40,'2024-02-10 00:00:00'), (50,'2024-02-20 00:00:00')) + ) + distributed by hash(a); + """ + sql """ + insert into t_multi_column_partition values(0,'2024-01-01 00:00:00',2),(1,'2024-01-01 00:00:00',2),(1,'2025-01-01 00:00:00',2), + (10,'2024-01-10 00:00:00',3),(10,'2024-01-11 00:00:00',200),(12,'2021-01-01 00:00:00',2), + (25,'2024-01-10 00:00:00',3),(20,'2024-01-11 00:00:00',200),(30,'2021-01-01 00:00:00',2), + (40,'2024-01-01 00:00:00',2),(40,'2024-01-31 00:00:00',2),(10,'2024-01-9 00:00:00',1000),(10,'2024-01-10 00:00:00',1000),(10,'2024-01-10 01:00:00',1000), + (2,'2023-01-10 01:00:00',1000) + """ + explain { + sql """select * from t_multi_column_partition where a=2 and date_trunc(dt,'day') <'2024-01-1 00:00:00';""" + contains ("partitions=1/5 (p0)") + } } \ No newline at end of file From 3bf1320a2ac103f98b6eb952a5038bca82102131 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 6 Nov 2024 15:34:37 +0800 Subject: [PATCH 2/3] put rangeMap in evaluate context:EvaluateRangeInput --- .../rules/OneRangePartitionEvaluator.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index fa0bd3d6781d2e..84a037171f32c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -93,7 +93,6 @@ public class OneRangePartitionEvaluator private final List> inputs; private final Map partitionSlotContainsNull; private final Map slotToType; - private final Map rangeMap = new HashMap<>(); /** OneRangePartitionEvaluator */ public OneRangePartitionEvaluator(long partitionId, List partitionSlots, @@ -174,10 +173,9 @@ public List> getOnePartitionInputs() { @Override public Expression evaluate(Expression expression, Map currentInputs) { - rangeMap.clear(); Map defaultColumnRanges = currentInputs.values().iterator().next().columnRanges; - rangeMap.putAll(defaultColumnRanges); - EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs)); + Map rangeMap = new HashMap<>(defaultColumnRanges); + EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs, rangeMap)); return result.result; } @@ -398,6 +396,7 @@ public EvaluateRangeResult visitIsNull(IsNull isNull, EvaluateRangeInput context public EvaluateRangeResult visitAnd(And and, EvaluateRangeInput context) { EvaluateRangeResult result = evaluateChildrenThenThis(and, context); result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1), + context.rangeMap, (leftRange, rightRange) -> leftRange.intersect(rightRange)); result = returnFalseIfExistEmptyRange(result); @@ -426,6 +425,7 @@ public EvaluateRangeResult visitOr(Or or, EvaluateRangeInput context) { result.childrenResult); } result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1), + context.rangeMap, (leftRange, rightRange) -> leftRange.union(rightRange)); return returnFalseIfExistEmptyRange(result); } @@ -438,8 +438,8 @@ public EvaluateRangeResult visitNot(Not not, EvaluateRangeInput context) { for (Map.Entry entry : result.childrenResult.get(0).columnRanges.entrySet()) { Expression expr = entry.getKey(); ColumnRange childRange = entry.getValue(); - ColumnRange partitionRange = rangeMap.containsKey(expr) - ? rangeMap.get(expr) : ColumnRange.all(); + ColumnRange partitionRange = context.rangeMap.containsKey(expr) + ? context.rangeMap.get(expr) : ColumnRange.all(); newRanges.put(expr, partitionRange.intersect(childRange.complete())); } result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult); @@ -563,6 +563,7 @@ private Map replaceExprRange(Map rangeMap, BiFunction mergeFunction) { Map leftRanges = left.columnRanges; @@ -624,7 +625,7 @@ public EvaluateRangeResult visitDateTrunc(DateTrunc dateTrunc, EvaluateRangeInpu if (partitionSlotContainsNull.containsKey(dateTruncChild)) { partitionSlotContainsNull.put(dateTrunc, true); } - return computeMonotonicFunctionRange(result); + return computeMonotonicFunctionRange(result, context.rangeMap); } @Override @@ -637,7 +638,7 @@ public EvaluateRangeResult visitDate(Date date, EvaluateRangeInput context) { if (partitionSlotContainsNull.containsKey(dateChild)) { partitionSlotContainsNull.put(date, true); } - return computeMonotonicFunctionRange(result); + return computeMonotonicFunctionRange(result, context.rangeMap); } @Override @@ -650,7 +651,7 @@ public EvaluateRangeResult visitConvertTz(ConvertTz convertTz, EvaluateRangeInpu if (partitionSlotContainsNull.containsKey(converTzChild)) { partitionSlotContainsNull.put(convertTz, true); } - return computeMonotonicFunctionRange(result); + return computeMonotonicFunctionRange(result, context.rangeMap); } private boolean isPartitionSlot(Slot slot) { @@ -682,10 +683,12 @@ private Map fillSlotRangesToInputs( /** EvaluateRangeInput */ public static class EvaluateRangeInput { - private Map slotToInput; + private final Map slotToInput; + private final Map rangeMap; - public EvaluateRangeInput(Map slotToInput) { + public EvaluateRangeInput(Map slotToInput, Map rangeMap) { this.slotToInput = slotToInput; + this.rangeMap = rangeMap; } } @@ -817,7 +820,8 @@ private List> commonComputeOnePartitionInputs() { return onePartitionInputs; } - private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result) { + private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result, + Map rangeMap) { Monotonic func = (Monotonic) result.result; if (rangeMap.containsKey(func)) { return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func, From cf9dd4c2c69f545c9544be458e9395f7d026722c Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 7 Nov 2024 12:16:51 +0800 Subject: [PATCH 3/3] fix regresion --- .../partition_prune/test_date_trunc_prune.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy index d6f40e39a8df68..c5eb3855247ad0 100644 --- a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy +++ b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy @@ -346,7 +346,7 @@ suite("test_date_trunc_prune") { partition p30 values [(30,'2024-01-31 00:00:00'), (40,'2024-02-10 00:00:00')), partition p40 values [(40,'2024-02-10 00:00:00'), (50,'2024-02-20 00:00:00')) ) - distributed by hash(a); + distributed by hash(a) properties("replication_num"="1"); """ sql """ insert into t_multi_column_partition values(0,'2024-01-01 00:00:00',2),(1,'2024-01-01 00:00:00',2),(1,'2025-01-01 00:00:00',2),