From 798d24f0213d2373a106e9f15f3188fa0dedad46 Mon Sep 17 00:00:00 2001 From: kakachen Date: Fri, 18 Apr 2025 19:48:16 +0800 Subject: [PATCH] [Fix](orc-reader-merge-io) Clear streams map when all row groups are filtered by row group stats, despite stripe stats remaining unf;5Diltered. --- be/src/apache-orc | 2 +- .../hive/test_orc_merge_io_input_streams.out | 7 +++ .../test_orc_merge_io_input_streams.groovy | 52 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/external_table_p2/hive/test_orc_merge_io_input_streams.out create mode 100644 regression-test/suites/external_table_p2/hive/test_orc_merge_io_input_streams.groovy diff --git a/be/src/apache-orc b/be/src/apache-orc index c9891b33063b32..18fb8e2c2888a3 160000 --- a/be/src/apache-orc +++ b/be/src/apache-orc @@ -1 +1 @@ -Subproject commit c9891b33063b32c72f3db6c61b6b59ba65444aee +Subproject commit 18fb8e2c2888a3518bf2bbd905f60772f4754739 diff --git a/regression-test/data/external_table_p2/hive/test_orc_merge_io_input_streams.out b/regression-test/data/external_table_p2/hive/test_orc_merge_io_input_streams.out new file mode 100644 index 00000000000000..fa812f2afc2260 --- /dev/null +++ b/regression-test/data/external_table_p2/hive/test_orc_merge_io_input_streams.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +0 + +-- !2 -- +60000 + diff --git a/regression-test/suites/external_table_p2/hive/test_orc_merge_io_input_streams.groovy b/regression-test/suites/external_table_p2/hive/test_orc_merge_io_input_streams.groovy new file mode 100644 index 00000000000000..8406bf0b42312b --- /dev/null +++ b/regression-test/suites/external_table_p2/hive/test_orc_merge_io_input_streams.groovy @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_orc_merge_io_input_streams", "p2,external,hive,external_remote,external_remote_hive") { + + String enabled = context.config.otherConfigs.get("enableExternalHudiTest") + //hudi hive use same catalog in p2. + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable test") + return; + } + + String props = context.config.otherConfigs.get("hudiEmrCatalog") + String hms_catalog_name = "test_orc_merge_io_input_streams" + + sql """drop catalog if exists ${hms_catalog_name};""" + sql """ + CREATE CATALOG IF NOT EXISTS ${hms_catalog_name} + PROPERTIES ( + ${props} + ,'hive.version' = '3.1.3' + ); + """ + + logger.info("catalog " + hms_catalog_name + " created") + sql """switch ${hms_catalog_name};""" + logger.info("switched to catalog " + hms_catalog_name) + sql """ use regression;""" + + sql """ set dry_run_query=true; """ + + qt_1 """ SELECT trace_id as trace_id_sub,created_time FROM test_orc_merge_io_input_streams_table + where dt = replace(date_sub('2025-04-16', 1), '-', '') and trace_id='1210647803'; """ + qt_2 """ select * from test_orc_merge_io_input_streams_table ; """ + + sql """drop catalog ${hms_catalog_name};""" +} +