From 0aa507a0d48d5efc9367f6eccecb425d0b5b5430 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 7 Aug 2024 16:24:11 +0800 Subject: [PATCH] [fix](mtmv) Fix rewrite fail when query direct external table without group by --- ...MaterializedViewFilterProjectScanRule.java | 13 +- .../mv/MaterializedViewFilterScanRule.java | 7 +- .../mv/MaterializedViewOnlyScanRule.java | 3 +- ...MaterializedViewProjectFilterScanRule.java | 13 +- .../mv/MaterializedViewProjectScanRule.java | 7 +- .../external_table/single_external_table.out | 17 +++ .../single_external_table.groovy | 116 ++++++++++++++++++ 7 files changed, 159 insertions(+), 17 deletions(-) create mode 100644 regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out create mode 100644 regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java index 613b356cdc579c..c828be694a945f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java @@ -19,8 +19,9 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -37,9 +38,11 @@ public class MaterializedViewFilterProjectScanRule extends AbstractMaterializedV @Override public List buildRules() { return ImmutableList.of( - logicalFilter(logicalProject(logicalOlapScan())).thenApplyMultiNoThrow(ctx -> { - LogicalFilter> root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_SCAN)); + logicalFilter(logicalProject(any().when(LogicalCatalogRelation.class::isInstance))) + .thenApplyMultiNoThrow( + ctx -> { + LogicalFilter> root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_SCAN)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java index 18f366354ce3e8..e7bbd28b51bd7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java @@ -19,8 +19,9 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import com.google.common.collect.ImmutableList; @@ -36,8 +37,8 @@ public class MaterializedViewFilterScanRule extends AbstractMaterializedViewScan @Override public List buildRules() { return ImmutableList.of( - logicalFilter(logicalOlapScan()).thenApplyMultiNoThrow(ctx -> { - LogicalFilter root = ctx.root; + logicalFilter(any().when(LogicalCatalogRelation.class::isInstance)).thenApplyMultiNoThrow(ctx -> { + LogicalFilter root = ctx.root; return rewrite(root, ctx.cascadesContext); }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_SCAN)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyScanRule.java index 97fe2153885737..24211ac8b08522 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyScanRule.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import com.google.common.collect.ImmutableList; @@ -34,7 +35,7 @@ public class MaterializedViewOnlyScanRule extends AbstractMaterializedViewScanRu @Override public List buildRules() { return ImmutableList.of( - logicalOlapScan().thenApplyMultiNoThrow(ctx -> { + any().when(LogicalCatalogRelation.class::isInstance).thenApplyMultiNoThrow(ctx -> { return rewrite(ctx.root, ctx.cascadesContext); }).toRule(RuleType.MATERIALIZED_VIEW_ONLY_SCAN)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java index d8a92ef59f292c..08abf6d6a59d97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java @@ -19,8 +19,9 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -37,9 +38,11 @@ public class MaterializedViewProjectFilterScanRule extends AbstractMaterializedV @Override public List buildRules() { return ImmutableList.of( - logicalProject(logicalFilter(logicalOlapScan())).thenApplyMultiNoThrow(ctx -> { - LogicalProject> root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_SCAN)); + logicalProject(logicalFilter(any().when(LogicalCatalogRelation.class::isInstance))) + .thenApplyMultiNoThrow( + ctx -> { + LogicalProject> root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_SCAN)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java index 72a656532cd790..1a51eb615b8b94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java @@ -19,7 +19,8 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import com.google.common.collect.ImmutableList; @@ -36,8 +37,8 @@ public class MaterializedViewProjectScanRule extends AbstractMaterializedViewSca @Override public List buildRules() { return ImmutableList.of( - logicalProject(logicalOlapScan()).thenApplyMultiNoThrow(ctx -> { - LogicalProject root = ctx.root; + logicalProject(any().when(LogicalCatalogRelation.class::isInstance)).thenApplyMultiNoThrow(ctx -> { + LogicalProject root = ctx.root; return rewrite(root, ctx.cascadesContext); }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_SCAN)); } diff --git a/regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out b/regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out new file mode 100644 index 00000000000000..5305ddb7e5cdcf --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/external_table/single_external_table.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +1 +2 +3 + +-- !query1_0_after -- +1 +2 +3 + +-- !query1_1_before -- +3 + +-- !query1_1_after -- +3 + diff --git a/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy new file mode 100644 index 00000000000000..30f3fe64b3f8d9 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy @@ -0,0 +1,116 @@ +package mv.external_table +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("single_external_table", "p0,external,hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test. then doesn't test mv rewrite") + return; + } + // prepare catalog + def suite_name = "single_external_table"; + def externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def hms_port = context.config.otherConfigs.get("hive2HmsPort") + def hive_catalog_name = "${suite_name}_catalog" + def hive_database = "${suite_name}_db" + def hive_table = "${suite_name}_orders" + + sql """drop catalog if exists ${hive_catalog_name}""" + sql """ + create catalog if not exists ${hive_catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + + sql """switch ${hive_catalog_name};""" + sql """drop table if exists ${hive_catalog_name}.${hive_database}.${hive_table}""" + sql """ drop database if exists ${hive_database}""" + sql """ create database ${hive_database}""" + sql """use ${hive_database}""" + sql """ + CREATE TABLE IF NOT EXISTS ${hive_table} ( + o_orderkey integer, + o_custkey integer, + o_orderstatus char(1), + o_totalprice decimalv3(15,2), + o_orderpriority char(15), + o_clerk char(15), + o_shippriority integer, + o_comment varchar(79), + o_orderdate date + ) ENGINE=hive + PARTITION BY list(o_orderdate)() + PROPERTIES ( + "replication_num" = "1", + "file_format"="orc", + "compression"="zlib" + ); + """ + + sql """insert into ${hive_catalog_name}.${hive_database}.${hive_table} values(1, 1, 'ok', 99.5, 'a', 'b', 1, 'yy', '2023-10-17');""" + sql """insert into ${hive_catalog_name}.${hive_database}.${hive_table} values(2, 2, 'ok', 109.2, 'c','d',2, 'mm', '2023-10-18');""" + sql """insert into ${hive_catalog_name}.${hive_database}.${hive_table} values(3, 3, 'ok', 99.5, 'a', 'b', 1, 'yy', '2023-10-19');""" + + // prepare table and data in olap + def internal_catalog = "internal" + def olap_db = context.config.getDbNameByFile(context.file) + + sql """switch ${internal_catalog};""" + sql "use ${olap_db};" + sql "SET enable_nereids_planner=true;" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject';" + sql "SET materialized_view_rewrite_enable_contain_external_table=true" + + + // single table without aggregate + def mv1_0 = """ + select o_custkey, o_orderdate + from ${hive_catalog_name}.${hive_database}.${hive_table}; + """ + def query1_0 = """ + select o_custkey + from ${hive_catalog_name}.${hive_database}.${hive_table}; + """ + order_qt_query1_0_before "${query1_0}" + check_mv_rewrite_success(olap_db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + + // single table filter without aggregate + def mv1_1 = """ + select o_custkey, o_orderdate + from ${hive_catalog_name}.${hive_database}.${hive_table} + where o_custkey > 1; + """ + def query1_1 = """ + select o_custkey + from ${hive_catalog_name}.${hive_database}.${hive_table} + where o_custkey > 2; + """ + order_qt_query1_1_before "${query1_1}" + check_mv_rewrite_success(olap_db, mv1_1, query1_1, "mv1_1") + order_qt_query1_1_after "${query1_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" + + + sql """drop table if exists ${hive_catalog_name}.${hive_database}.${hive_table}""" + sql """drop database if exists ${hive_catalog_name}.${hive_database}""" + sql """drop catalog if exists ${hive_catalog_name}""" +}