diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 1a66eda2ad3eed..743003dca6bb81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -50,7 +50,6 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Repeat; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; @@ -155,7 +154,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode, materializationContext.getShuttledExprToScanExprMapping(), viewToQuerySlotMapping)); } - return doRewriteQueryByView(queryStructInfo, + return aggregateRewriteByView(queryStructInfo, viewToQuerySlotMapping, queryTopPlanAndAggPair, tempRewritedPlan, @@ -167,7 +166,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode, /** * Aggregate function and group by expression rewrite impl */ - protected LogicalAggregate doRewriteQueryByView( + protected LogicalAggregate aggregateRewriteByView( StructInfo queryStructInfo, SlotMapping viewToQuerySlotMapping, Pair> queryTopPlanAndAggPair, @@ -426,12 +425,11 @@ private boolean isGroupByEquals(Pair> queryTopPlanA LogicalAggregate queryAggregate = queryTopPlanAndAggPair.value(); LogicalAggregate viewAggregate = viewTopPlanAndAggPair.value(); - Set queryGroupShuttledExpression = new HashSet<>(ExpressionUtils.shuttleExpressionWithLineage( + Set queryGroupByShuttledExpression = new HashSet<>(ExpressionUtils.shuttleExpressionWithLineage( queryAggregate.getGroupByExpressions(), queryTopPlan, queryStructInfo.getTableBitSet())); // try to eliminate group by dimension by function dependency if group by expression is not in query Map viewShuttledExpressionQueryBasedToGroupByExpressionMap = new HashMap<>(); - Map groupByExpressionToViewShuttledExpressionQueryBasedMap = new HashMap<>(); List viewGroupByExpressions = viewAggregate.getGroupByExpressions(); List viewGroupByShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( viewGroupByExpressions, viewTopPlan, viewStructInfo.getTableBitSet()); @@ -443,43 +441,40 @@ private boolean isGroupByEquals(Pair> queryTopPlanA viewToQuerySlotMapping.toSlotReferenceMap()); viewShuttledExpressionQueryBasedToGroupByExpressionMap.put(viewGroupExpressionQueryBased, viewExpression); - groupByExpressionToViewShuttledExpressionQueryBasedMap.put(viewExpression, - viewGroupExpressionQueryBased - ); } - if (queryGroupShuttledExpression.equals(viewShuttledExpressionQueryBasedToGroupByExpressionMap.keySet())) { + if (queryGroupByShuttledExpression.equals(viewShuttledExpressionQueryBasedToGroupByExpressionMap.keySet())) { // return true, if equals directly return true; } - - boolean isGroupByEquals = false; - // check is equals by group by eliminate - isGroupByEquals |= isGroupByEqualsAfterGroupByEliminate(queryGroupShuttledExpression, - viewShuttledExpressionQueryBasedToGroupByExpressionMap, - groupByExpressionToViewShuttledExpressionQueryBasedMap, - viewAggregate, - cascadesContext); - // check is equals by equal filter eliminate - Optional> filterOptional = tempRewrittenPlan.collectFirst(LogicalFilter.class::isInstance); - if (!filterOptional.isPresent()) { - return isGroupByEquals; - } - isGroupByEquals |= isGroupByEqualsAfterEqualFilterEliminate( + // Check is equals by equal filter eliminate + return isGroupByEqualsByFunctionDependency( (LogicalPlan) tempRewrittenPlan, - queryGroupShuttledExpression, + queryGroupByShuttledExpression, viewShuttledExpressionQueryBasedToGroupByExpressionMap, materializationContext); - return isGroupByEquals; } /** - * Check group by is equals by equal filter eliminate - * For example query is select a, b, c from t1 where a = 1 and d = 'xx' group by a, b, c; - * mv is select a, b, c, d from t1 group by a, b, c, d; - * the group by expression between query and view is equals after equal filter eliminate - * should not aggregate roll up + * Check group by is equals by uniform function dependency + * For example query is: + * select + * a, b, c from t1 + * where a = 1 and d = 'xx' + * group by a, b, c; + * mv is : + * select a, b, c, d + * from t1 + * group by a, b, c, d; + * After group by key eliminate, the query group by is b, c + * but mv is group by a, b, c, d, the group by a and d of mv is more dimensions than the query + * But in tempRewrittenPlan is as following: + * select * + * from mv + * where a = 1 and d = 'xx' + * We can get group by a and d is uniform by function dependency info, + * so the group by expression between query and view is equals, should not aggregate roll up * */ - private static boolean isGroupByEqualsAfterEqualFilterEliminate( + private static boolean isGroupByEqualsByFunctionDependency( LogicalPlan tempRewrittenPlan, Set queryGroupShuttledExpression, Map viewShuttledExprQueryBasedToViewGroupByExprMap, @@ -541,42 +536,42 @@ private static boolean isGroupByEqualsAfterEqualFilterEliminate( * viewAggregate * 4. check the viewAggregate group by expression is equals queryAggregate expression or not */ - private static boolean isGroupByEqualsAfterGroupByEliminate(Set queryGroupShuttledExpression, + private static boolean isGroupByEqualsAfterGroupByEliminate(Set queryGroupByShuttledExpression, Map viewShuttledExpressionQueryBasedToGroupByExpressionMap, Map groupByExpressionToViewShuttledExpressionQueryBasedMap, LogicalAggregate viewAggregate, CascadesContext cascadesContext) { - List projects = new ArrayList<>(); - // construct projects query used by view group expressions - for (Expression expression : queryGroupShuttledExpression) { + List viewProjects = new ArrayList<>(); + // construct viewProjects query used by view group expressions + for (Expression expression : queryGroupByShuttledExpression) { Expression chosenExpression = viewShuttledExpressionQueryBasedToGroupByExpressionMap.get(expression); if (chosenExpression == null) { return false; } - projects.add(chosenExpression instanceof NamedExpression + viewProjects.add(chosenExpression instanceof NamedExpression ? (NamedExpression) chosenExpression : new Alias(chosenExpression)); } - LogicalProject> project = new LogicalProject<>(projects, viewAggregate); + LogicalProject> viewProject = new LogicalProject<>(viewProjects, viewAggregate); // try to eliminate view group by expression which is not in query group by expression Plan rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext, childContext -> { Rewriter.getCteChildrenRewriter(childContext, ImmutableList.of(Rewriter.topDown(new EliminateGroupByKey()))).execute(); return childContext.getRewritePlan(); - }, project, project); + }, viewProject, viewProject); - Optional> aggreagateOptional = + Optional> viewAggreagateOptional = rewrittenPlan.collectFirst(LogicalAggregate.class::isInstance); - if (!aggreagateOptional.isPresent()) { + if (!viewAggreagateOptional.isPresent()) { return false; } // check result after view group by eliminate by functional dependency - List viewEliminatedGroupByExpressions = aggreagateOptional.get().getGroupByExpressions(); - if (viewEliminatedGroupByExpressions.size() != queryGroupShuttledExpression.size()) { + List viewEliminatedGroupByExpressions = viewAggreagateOptional.get().getGroupByExpressions(); + if (viewEliminatedGroupByExpressions.size() != queryGroupByShuttledExpression.size()) { return false; } Set viewGroupShuttledExpressionQueryBased = new HashSet<>(); - for (Expression viewExpression : aggreagateOptional.get().getGroupByExpressions()) { + for (Expression viewExpression : viewAggreagateOptional.get().getGroupByExpressions()) { Expression viewExpressionQueryBased = groupByExpressionToViewShuttledExpressionQueryBasedMap.get(viewExpression); if (viewExpressionQueryBased == null) { @@ -584,7 +579,7 @@ private static boolean isGroupByEqualsAfterGroupByEliminate(Set quer } viewGroupShuttledExpressionQueryBased.add(viewExpressionQueryBased); } - return queryGroupShuttledExpression.equals(viewGroupShuttledExpressionQueryBased); + return queryGroupByShuttledExpression.equals(viewGroupShuttledExpressionQueryBased); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java index 7107238a309219..45514f6fb15e6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java @@ -124,7 +124,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf () -> String.format("query aggregate = %s", queryAggregate.treeString())); return null; } - return doRewriteQueryByView(queryStructInfo, + return aggregateRewriteByView(queryStructInfo, viewToQuerySlotMapping, queryTopPlanAndAggPair, tempRewritedPlan, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java index a96c272521aef7..190080424511c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java @@ -62,12 +62,12 @@ public abstract Function doRollup( /** * Extract the function arguments by functionWithAny pattern * Such as functionWithAny def is bitmap_union(to_bitmap(Any.INSTANCE)), - * actualFunction is bitmap_union(to_bitmap(case when a = 5 then 1 else 2 end)) + * actualExpression is bitmap_union(to_bitmap(case when a = 5 then 1 else 2 end)) * after extracting, the return argument is: case when a = 5 then 1 else 2 end */ - protected static List extractArguments(Expression functionWithAny, Function actualFunction) { + protected static List extractArguments(Expression functionWithAny, Expression actualExpression) { Set exprSetToRemove = functionWithAny.collectToSet(expr -> !(expr instanceof Any)); - return actualFunction.collectFirst(expr -> + return actualExpression.collectFirst(expr -> exprSetToRemove.stream().noneMatch(exprToRemove -> exprToRemove.equals(expr))) .map(expr -> ImmutableList.of((Expression) expr)).orElse(ImmutableList.of()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/ContainDistinctFunctionRollupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/ContainDistinctFunctionRollupHandler.java index 4d9e6810ce4521..63b148dfcd7698 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/ContainDistinctFunctionRollupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/ContainDistinctFunctionRollupHandler.java @@ -76,6 +76,7 @@ public boolean canRollup(AggregateFunction queryAggregateFunction, Set mvExpressionsQueryBased = mvExprToMvScanExprQueryBased.keySet(); Set aggregateFunctionParamSlots = queryAggregateFunctionShuttled.collectToSet(Slot.class::isInstance); if (aggregateFunctionParamSlots.stream().anyMatch(slot -> !mvExpressionsQueryBased.contains(slot))) { + // If query use any slot not in view, can not roll up return false; } return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/DirectRollupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/DirectRollupHandler.java index fe443afbc99536..ae94eda115c681 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/DirectRollupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/DirectRollupHandler.java @@ -52,9 +52,7 @@ public boolean canRollup( && viewExpression instanceof NullableAggregateFunction ? ((NullableAggregateFunction) queryAggregateFunctionShuttled).equalsIgnoreNullable(viewExpression) : queryAggregateFunctionShuttled.equals(viewExpression); - return isEquals && MappingRollupHandler.AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.keySet().stream() - .noneMatch(aggFunction -> aggFunction.equals(queryAggregateFunction)) - && !(queryAggregateFunction instanceof Combinator); + return isEquals && !(queryAggregateFunction instanceof Combinator); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/MappingRollupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/MappingRollupHandler.java index f3f81235f3cfcb..5ed661dd51a7e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/MappingRollupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/MappingRollupHandler.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion; import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; @@ -132,6 +133,9 @@ public class MappingRollupHandler extends AggFunctionRollUpHandler { AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new HllUnion(Any.INSTANCE), new HllUnion(Any.INSTANCE)); + // support roll up when any_value + AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new AnyValue(Any.INSTANCE), + Any.INSTANCE); } @Override @@ -141,24 +145,20 @@ public boolean canRollup(AggregateFunction queryAggregateFunction, Map mvExprToMvScanExprQueryBasedMap) { // handle complex functions roll up by mapping and combinator expression // eg: query is count(distinct param), mv sql is bitmap_union(to_bitmap(param)) - Expression viewExpression = mvExprToMvScanExprQueryBasedPair.key(); - if (!super.canRollup(queryAggregateFunction, queryAggregateFunctionShuttled, - mvExprToMvScanExprQueryBasedPair, mvExprToMvScanExprQueryBasedMap)) { - return false; - } - Function viewFunction = (Function) viewExpression; - for (Map.Entry> equivalentFunctionEntry : + Expression actualViewExpression = mvExprToMvScanExprQueryBasedPair.key(); + for (Map.Entry> queryToViewEquivalentMapEntry : AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.asMap().entrySet()) { - if (equivalentFunctionEntry.getKey().equals(queryAggregateFunction)) { - // check is have equivalent function or not - for (Expression equivalentFunction : equivalentFunctionEntry.getValue()) { - if (!Any.equals(equivalentFunction, viewFunction)) { + if (queryToViewEquivalentMapEntry.getKey().equals(queryAggregateFunction)) { + for (Expression mappedViewExpression : queryToViewEquivalentMapEntry.getValue()) { + if (!Any.equals(mappedViewExpression, actualViewExpression)) { + // check the mapping view expression is equivalent with actual view expression continue; } // check param in query function is same as the view function - List viewFunctionArguments = extractArguments(equivalentFunction, viewFunction); - List queryFunctionArguments = - extractArguments(equivalentFunctionEntry.getKey(), queryAggregateFunction); + List viewFunctionArguments = extractArguments(mappedViewExpression, + actualViewExpression); + List queryFunctionArguments = extractArguments(queryToViewEquivalentMapEntry.getKey(), + queryAggregateFunction); // check argument size,we only support roll up function which has only one argument currently if (queryFunctionArguments.size() != 1 || viewFunctionArguments.size() != 1) { continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AnyValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AnyValue.java index 50cdacc132b3d4..e85c2ce551c230 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AnyValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AnyValue.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.coercion.AnyDataType; @@ -33,7 +34,7 @@ * AggregateFunction 'any_value'. This class is generated by GenerateFunction. */ public class AnyValue extends NullableAggregateFunction - implements UnaryExpression, ExplicitlyCastableSignature { + implements UnaryExpression, ExplicitlyCastableSignature, RollUpTrait { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.retArgType(0).args(AnyDataType.INSTANCE_WITHOUT_INDEX) @@ -80,4 +81,14 @@ public R accept(ExpressionVisitor visitor, C context) { public List getSignatures() { return SIGNATURES; } + + @Override + public Function constructRollUp(Expression param, Expression... varParams) { + return new AnyValue(this.distinct, this.alwaysNullable, param); + } + + @Override + public boolean canRollUp() { + return true; + } } diff --git a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.out new file mode 100644 index 00000000000000..9c69bcde1ab846 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.out @@ -0,0 +1,73 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +3 o +3 o +3 o +3 o +4 o + +-- !query1_0_after -- +3 o +3 o +3 o +3 o +4 o + +-- !query2_0_before -- +3 o +3 o +3 o +3 o +4 o + +-- !query2_0_after -- +3 o +3 o +3 o +3 o +4 o + +-- !query3_0_before -- +3 3 o + +-- !query3_0_after -- +3 3 o + +-- !query4_0_before -- +3 o + +-- !query4_0_after -- +3 o + +-- !query5_0_before -- +3 3 o + +-- !query5_0_after -- +3 3 o + +-- !query5_1_before -- +3 3 o + +-- !query5_1_after -- +3 3 o + +-- !query6_0_before -- +3 o + +-- !query6_0_after -- +3 o + +-- !query6_1_before -- +3 o +3 o +3 o +3 o +4 o + +-- !query6_1_after -- +3 o +3 o +3 o +3 o +4 o + diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.groovy new file mode 100644 index 00000000000000..44815c8a1427f0 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.groovy @@ -0,0 +1,417 @@ +package mv.agg_with_roll_up +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("any_value_roll_up") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders_2 + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders_2 ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem_2 + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem_2 ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp_2 + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp_2 ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem_2 values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); + """ + + sql """ + insert into orders_2 values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + insert into partsupp_2 values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + sql """analyze table partsupp_2 with sync""" + sql """analyze table lineitem_2 with sync""" + sql """analyze table orders_2 with sync""" + + + // mv has any value, query also has any value + def mv1_0 = """ + select + o_orderdate, + l_partkey, + l_suppkey, + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + o_orderdate, + l_partkey, + l_suppkey; + """ + def query1_0 = """ + select + l_suppkey, + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + o_orderdate, + l_suppkey; + """ + order_qt_query1_0_before "${query1_0}" + async_mv_rewrite_success(db, mv1_0, query1_0, "any_mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS any_mv1_0""" + + + // query has any value, mv doesn't have + def mv2_0 = """ + select + o_orderdate, + l_partkey, + l_suppkey, + o_orderstatus + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + o_orderstatus, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query2_0 = """ + select + l_suppkey, + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + o_orderdate, + l_suppkey; + """ + order_qt_query2_0_before "${query2_0}" + async_mv_rewrite_success(db, mv2_0, query2_0, "any_mv2_0") + order_qt_query2_0_after "${query2_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS any_mv2_0""" + + + // mv has any value, query also has any value, with same filter + def mv3_0 = """ + select + l_suppkey, + o_orderdate, + l_partkey, + o_orderstatus, + any_value(l_suppkey), + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where l_partkey = 3 and o_orderstatus = 'o' + group by + o_orderstatus, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query3_0 = """ + select + l_suppkey, + any_value(l_suppkey), + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where l_partkey = 3 and o_orderstatus = 'o' + group by + o_orderdate, + l_suppkey; + """ + order_qt_query3_0_before "${query3_0}" + async_mv_rewrite_success(db, mv3_0, query3_0, "any_mv3_0") + order_qt_query3_0_after "${query3_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS any_mv3_0""" + + + // query has any value, mv doesn't have, with same filter + def mv4_0 = """ + select + o_orderdate, + l_partkey, + l_suppkey, + o_orderstatus + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where l_partkey = 3 and o_orderstatus = 'o' + group by + o_orderstatus, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query4_0 = """ + select + any_value(l_suppkey), + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where l_partkey = 3 and o_orderstatus = 'o' + group by + o_orderdate, + l_suppkey; + """ + order_qt_query4_0_before "${query4_0}" + async_mv_rewrite_success(db, mv4_0, query4_0, "any_mv4_0") + order_qt_query4_0_after "${query4_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS any_mv4_0""" + + + // mv has any value, query also has any value, with different filter + def mv5_0 = """ + select + o_orderdate, + any_value(l_partkey), + o_orderstatus, + l_suppkey, + any_value(l_suppkey), + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where l_partkey = 3 + group by + o_orderstatus, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query5_0 = """ + select + any_value(l_partkey), + any_value(l_suppkey), + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where l_partkey = 3 and o_orderstatus = 'o' + group by + o_orderdate, + l_suppkey; + """ + order_qt_query5_0_before "${query5_0}" + async_mv_rewrite_success(db, mv5_0, query5_0, "any_mv5_0") + order_qt_query5_0_after "${query5_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS any_mv5_0""" + + + def mv5_1 = """ + select + o_orderdate, + any_value(l_partkey), + any_value(l_suppkey), + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where l_partkey = 3 and o_orderstatus = 'o' + group by + o_orderstatus, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query5_1 = """ + select + l_suppkey, + any_value(l_suppkey), + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where l_partkey = 3 + group by + o_orderdate, + l_suppkey; + """ + order_qt_query5_1_before "${query5_1}" + async_mv_rewrite_fail(db, mv5_1, query5_1, "any_mv5_1") + order_qt_query5_1_after "${query5_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS any_mv5_1""" + + + // query has any value, mv doesn't have, with different filter + def mv6_0 = """ + select + o_orderdate, + l_partkey, + l_suppkey, + o_orderstatus + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where o_orderstatus = 'o' + group by + o_orderstatus, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query6_0 = """ + select + any_value(l_suppkey), + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where l_partkey = 3 and o_orderstatus = 'o' + group by + o_orderdate, + l_suppkey; + """ + order_qt_query6_0_before "${query6_0}" + async_mv_rewrite_success(db, mv6_0, query6_0, "any_mv6_0") + order_qt_query6_0_after "${query6_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS any_mv6_0""" + + + def mv6_1 = """ + select + o_orderdate, + l_partkey, + l_suppkey, + o_orderstatus + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where l_partkey = 3 and o_orderstatus = 'o' + group by + o_orderstatus, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query6_1 = """ + select + l_suppkey, + any_value(o_orderstatus) + from lineitem_2 + left join orders_2 + on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where o_orderstatus = 'o' + group by + o_orderdate, + l_suppkey; + """ + order_qt_query6_1_before "${query6_1}" + async_mv_rewrite_fail(db, mv6_1, query6_1, "any_mv6_1") + order_qt_query6_1_after "${query6_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS any_mv6_1""" + +}