From c9dbb44196fab51047c8f843e006214b001da21a Mon Sep 17 00:00:00 2001 From: chenqi Date: Thu, 26 Oct 2023 20:34:00 +0800 Subject: [PATCH] [Fix](orc-reader) Fix orc decimal128 scale issue. --- be/src/apache-orc | 2 +- .../scripts/create_preinstalled_table.hql | 19 ++++++++++++++++++ .../orc_decimal_table/orc_decimal_table | Bin 0 -> 1169 bytes .../external_table_p0/hive/test_hive_orc.out | 18 +++++++++++++++++ .../hive/test_hive_orc.groovy | 10 +++++++++ 5 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_decimal_table/orc_decimal_table diff --git a/be/src/apache-orc b/be/src/apache-orc index a7c0af50f8ca8f..e52ba9094dd8fa 160000 --- a/be/src/apache-orc +++ b/be/src/apache-orc @@ -1 +1 @@ -Subproject commit a7c0af50f8ca8ff7cddaf8675473a037f8b13143 +Subproject commit e52ba9094dd8fa3e86e6043e42d6d561fe092d7c diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql index d7e93bb74ff844..2fbdbbe6dbcf46 100644 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql @@ -1743,5 +1743,24 @@ LOCATION msck repair table parquet_timestamp_nanos; +CREATE TABLE `orc_decimal_table`( + id INT, + decimal_col1 DECIMAL(8, 4), + decimal_col2 DECIMAL(18, 6), + decimal_col3 DECIMAL(38, 12), + decimal_col4 DECIMAL(9, 0), + decimal_col5 DECIMAL(27, 9), + decimal_col6 DECIMAL(9, 0)) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +LOCATION + '/user/doris/preinstalled_data/orc_table/orc_decimal_table'; + +msck repair table orc_decimal_table; + show tables; diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_decimal_table/orc_decimal_table b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_decimal_table/orc_decimal_table new file mode 100644 index 0000000000000000000000000000000000000000..c63a5e8ea37f22b6c871730f59665ccb4b010f93 GIT binary patch literal 1169 zcmeYdau#G@;9?VE;b0A5&;~MvxtJLk7=(B@ScTX)m;|^aVg@#b zGN9T|Pr)t~2fEZ6WL+lQItyq1W0NKs!d%K=e1^y8%$J_CybdR>C^axbY`pwn);L~&?%iW|41Sxm9K66F!o~nnz!S^B@M7z}d3XPrzxX=;$e$~_ z559kMXyV@^zc)-)opW>Bv5j4SPHxz=X)z;I6$3v5gCC281cL;FI0J*ksx9|QL0AH8 z8iP9n!>X1y-(R_-XkGQE>Fn1_XSXp!CBa4r2>{c938DsKK9tP1jCXW_z z@`uV?DF{w)|Mom2VD_z$vLmPZ_6qxzIVEbVZToh2!^XAm`&!>#(5lQ(um8?e!N`!> zXU*T_AkcELLoQl^xwHTBj09&@%Ww0J9Jw^>*pXxV-{lvID{;jY-rPGoKkb2O>&%YX z9MhEio+s^Udb_bxj{AgbkM)cdCMyGGPtu>c;Nz+*af+U137$o1p2zlI`x$UKpsf4M zg&n&tU1NWv@Rljj(uQ-laoC@v!wcWUHhJ$-wVP~u4)^?C64@72pd^3Ot6att_Tpt0%W%|q2bS45bKCjH#u z6YUh4tT#nStpITl(ejLXRlJ%tv1+n zGk)f7li$<*xJ-U;EfQ<&G&Q@oSK-*dR||H0H8|SqcjM92U00`MJJ0-+zro*ad3&PI zn@c-753RapcfxJvtT};~7Oi=!x_rlF_u{@hv+w2i_&F9b2{1}DGzcg$G4N>YXEb1M M`ohfYALJ|v0PR=Z=Kufz literal 0 HcmV?d00001 diff --git a/regression-test/data/external_table_p0/hive/test_hive_orc.out b/regression-test/data/external_table_p0/hive/test_hive_orc.out index 4f1841c7d31fe7..aff1b185cfb2e6 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_orc.out +++ b/regression-test/data/external_table_p0/hive/test_hive_orc.out @@ -83,6 +83,24 @@ tablets tinyint_col 179 182 182 187 183 181 177 183 177 187 183 202 202 186 528 -- !only_partition_col -- 3600 3600 +-- !decimals1 -- +1 1.1234 12.123456 123.123456789876 12 1234.123456789 123 +2 1234.1234 123456789123.123456 12345678912345678912345678.123456789876 123456789 123456789123456789.123456780 987654321 +3 1234.0000 123456789123.000000 12345678912345678912345678.000000000000 123456789 123456789123456789.000000000 987654321 +4 0.0000 0.000000 123.123456789876 12 0E-9 123 +5 1.1234 12.123456 0E-12 0 1234.123456789 0 + +-- !decimals2 -- +3 1234.0000 123456789123.000000 12345678912345678912345678.000000000000 123456789 123456789123456789.000000000 987654321 + +-- !decimals3 -- +1 1.1234 12.123456 123.123456789876 12 1234.123456789 123 +2 1234.1234 123456789123.123456 12345678912345678912345678.123456789876 123456789 123456789123456789.123456780 987654321 + +-- !decimals4 -- +4 0.0000 0.000000 123.123456789876 12 0E-9 123 +5 1.1234 12.123456 0E-12 0 1234.123456789 0 + -- !select_top50 -- 4 55 999742610 400899305488827731 false 6.5976813E8 7.8723304616937395E17 \N base tennis pit vertical friday 2022-08-19T07:29:58 \N tablets smallint_col 2019-02-07 [7.53124931825377e+17] ["NbSSBtwzpxNSkkwga"] tablets smallint_col 2 49 999613702 105493714032727452 \N 6.3322381E8 9.8642324410240179E17 Unveil bright recruit participate. Suspect impression camera mathematical revelation. Fault live2 elbow debt west hydrogen current. how literary 2022-09-03T17:20:21 481707.1065 tablets boolean_col 2020-01-12 [] ["HoMrAnn", "wteEFvIwoZsVpVQdscMb", NULL, "zcGFmv", "kGEBBckbMtX", "hrEtCGFdPWZK"] tablets boolean_col diff --git a/regression-test/suites/external_table_p0/hive/test_hive_orc.groovy b/regression-test/suites/external_table_p0/hive/test_hive_orc.groovy index e971d823405700..b6bccfb286c249 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_orc.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_orc.groovy @@ -66,6 +66,14 @@ suite("test_hive_orc", "all_types,p0,external,hive,external_docker,external_dock qt_only_partition_col """select count(p1_col), count(p2_col) from orc_all_types;""" } + // decimals + def decimals = { + qt_decimals1 """select * from orc_decimal_table order by id;""" + qt_decimals2 """select * from orc_decimal_table where id = 3 order by id;""" + qt_decimals3 """select * from orc_decimal_table where id < 3 order by id;""" + qt_decimals4 """select * from orc_decimal_table where id > 3 order by id;""" + } + String enabled = context.config.otherConfigs.get("enableHiveTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { try { @@ -86,6 +94,7 @@ suite("test_hive_orc", "all_types,p0,external,hive,external_docker,external_dock search_in_int() search_mix() only_partition_col() + decimals() sql """drop catalog if exists ${catalog_name}""" @@ -103,3 +112,4 @@ suite("test_hive_orc", "all_types,p0,external,hive,external_docker,external_dock } } } +