From 831cb4ff6168cdef4758f4e977ecad234ecc8ce0 Mon Sep 17 00:00:00 2001 From: daidai Date: Mon, 14 Jul 2025 11:05:29 +0800 Subject: [PATCH 1/3] [fix](function)fix month=0 of from_iso8601_date function. (#53050) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Related PR: #40695 Problem Summary: pr #40695 introduced the function `from_iso8601_date`, which parses the string to get year、mouth、 day, and sets the date value through the `set_time_unit` function. Since `set_time_unit` lacks some judgment on mouth, it may get an illegal date in the end, which may cause core in debug mode. sql : `select from_iso8601_date('2023-00-01');` ``` F20250709 09:50:14.366984 3587796 vdatetime_value.h:1222] Check failed: date_v2_value_.month_ != 0 *** Check failure stack trace: *** @ 0x559bd7050d96 google::LogMessage::SendToLog() @ 0x559bd704d7e0 google::LogMessage::Flush() @ 0x559bd70515d9 google::LogMessageFatal::~LogMessageFatal() @ 0x559bc725a570 doris::DateV2Value<>::set_time_unit<>() @ 0x559bc7257380 doris::vectorized::FromIso8601DateV2::execute() @ 0x559bc7255a88 doris::vectorized::FunctionOtherTypesToDateType<>::execute_impl() @ 0x559bc09e0781 doris::vectorized::DefaultExecutable::execute_impl() @ 0x559bc423aa20 doris::vectorized::PreparedFunctionImpl::_execute_skipped_constant_deal() @ 0x559bc4234938 doris::vectorized::PreparedFunctionImpl::execute_without_low_cardinality_columns() @ 0x559bc4233f42 doris::vectorized::PreparedFunctionImpl::default_implementation_for_nulls() @ 0x559bc423a773 doris::vectorized::PreparedFunctionImpl::_execute_skipped_constant_deal() @ 0x559bc4234938 doris::vectorized::PreparedFunctionImpl::execute_without_low_cardinality_columns() @ 0x559bc4234a57 doris::vectorized::PreparedFunctionImpl::execute() ``` --- be/src/vec/runtime/vdatetime_value.h | 7 +- be/test/vec/function/function_time_test.cpp | 73 +++++++++++++++++++ .../test_from_iso8601_date.out | 52 +++++++++++++ .../test_from_iso8601_date.groovy | 63 ++++++++++++++++ 4 files changed, 194 insertions(+), 1 deletion(-) diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index 12e961f5fa91bc..11dfb6e796bb0f 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -1211,13 +1211,18 @@ class DateV2Value { } date_v2_value_.year_ = val; } else if constexpr (unit == TimeUnit::MONTH) { - if (val > MAX_MONTH) [[unlikely]] { + DCHECK(date_v2_value_.year_ <= MAX_YEAR); + if (val > MAX_MONTH || val == 0) [[unlikely]] { return false; } date_v2_value_.month_ = val; } else if constexpr (unit == TimeUnit::DAY) { + DCHECK(date_v2_value_.year_ <= MAX_YEAR); DCHECK(date_v2_value_.month_ <= MAX_MONTH); DCHECK(date_v2_value_.month_ != 0); + if (val == 0) [[unlikely]] { + return false; + } if (val > S_DAYS_IN_MONTH[date_v2_value_.month_] && !(is_leap(date_v2_value_.year_) && date_v2_value_.month_ == 2 && val == 29)) { return false; diff --git a/be/test/vec/function/function_time_test.cpp b/be/test/vec/function/function_time_test.cpp index 1ea010e4afdbb3..0aa96c62961b8b 100644 --- a/be/test/vec/function/function_time_test.cpp +++ b/be/test/vec/function/function_time_test.cpp @@ -1892,4 +1892,77 @@ TEST(VTimestampFunctionsTest, next_day_test) { data_set)); } } + +TEST(VTimestampFunctionsTest, from_iso8601_date) { + std::string func_name = "from_iso8601_date"; + InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR}; + + DataSet data_set = { + {{std::string("2020-01-01")}, std::string("2020-01-01")}, + {{std::string("2020-01-01")}, std::string("2020-01-01")}, + {{std::string("-1")}, Null()}, + {{std::string("2025-07-11")}, std::string("2025-07-11")}, + {{std::string("2024-02-29")}, std::string("2024-02-29")}, + {{std::string("2025-02-29")}, Null()}, + {{std::string("2025-13-01")}, Null()}, + {{std::string("2020-W10")}, std::string("2020-03-02")}, + {{std::string("2025-W28")}, std::string("2025-07-07")}, + {{std::string("2025-W53")}, std::string("2025-12-29")}, + {{std::string("2025-W00")}, Null()}, + {{std::string("2020-123")}, std::string("2020-05-02")}, + {{std::string("2025-192")}, std::string("2025-07-11")}, + {{std::string("2024-366")}, std::string("2024-12-31")}, + {{std::string("2025-366")}, Null()}, + {{std::string("2025-000")}, std::string("2024-12-31")}, + {{std::string("2025/07/11")}, Null()}, + {{std::string("25-07-11")}, Null()}, + {{std::string("2025-7-11")}, Null()}, + {{std::string("invalid-date")}, Null()}, + {{std::string("2025-07-11T12:34:56")}, Null()}, + {{std::string("-1")}, Null()}, + {{std::string("9999-12-31")}, std::string("9999-12-31")}, + {{std::string("10000-01-01")}, Null()}, + {{std::string("0001-01-01")}, std::string("0001-01-01")}, + {{std::string("0000-12-31")}, std::string("0000-12-31")}, + {{std::string("-0001-01-01")}, Null()}, + {{std::string("2025-01-01")}, std::string("2025-01-01")}, + {{std::string("2025-12-31")}, std::string("2025-12-31")}, + {{std::string("2025-00-01")}, Null()}, + {{std::string("2025-13-01")}, Null()}, + {{std::string("2025--01-01")}, Null()}, + {{std::string("2025-01-31")}, std::string("2025-01-31")}, + {{std::string("2025-04-30")}, std::string("2025-04-30")}, + {{std::string("2025-02-28")}, std::string("2025-02-28")}, + {{std::string("2024-02-29")}, std::string("2024-02-29")}, + {{std::string("2025-01-32")}, Null()}, + {{std::string("2025-04-31")}, Null()}, + {{std::string("2025-02-29")}, Null()}, + {{std::string("2025-02-30")}, Null()}, + {{std::string("2025-01-00")}, Null()}, + {{std::string("2025-01--01")}, Null()}, + {{std::string("2000-02-29")}, std::string("2000-02-29")}, + {{std::string("2024-02-29")}, std::string("2024-02-29")}, + {{std::string("1900-02-29")}, Null()}, + {{std::string("2100-02-29")}, Null()}, + {{std::string("2025-02-29")}, Null()}, + {{std::string("-2025-01-01")}, Null()}, + {{std::string("2025--07-01")}, Null()}, + {{std::string("2025-07--01")}, Null()}, + {{std::string("")}, Null()}, + {{std::string("2025")}, std::string("2025-01-01")}, + {{std::string("2025-07")}, std::string("2025-07-01")}, + {{std::string("99999-01-01")}, Null()}, + {{std::string("2025-123-01")}, Null()}, + {{std::string("2025-01-123")}, Null()}, + {{std::string("2025/01/01")}, Null()}, + {{std::string("2025.01.01")}, Null()}, + {{std::string("2025-01-01X")}, Null()}, + {{std::string("2025--01--01")}, Null()}, + {{std::string("abcd-01-01")}, Null()}, + {{std::string("2025-ab-01")}, Null()}, + {{std::string("2025-01-ab")}, Null()}, + }; + + static_cast(check_function(func_name, input_types, data_set)); +} } // namespace doris::vectorized diff --git a/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out b/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out index 43a4f0bd496c38..93b8bda36e1f92 100644 --- a/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out +++ b/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out @@ -242,3 +242,55 @@ \N \N +-- !test_99 -- +1 2023-01-01 +2 2023-12-31 +3 2020-02-29 +4 1970-01-01 +5 9999-12-31 +6 1000-01-01 +7 \N +8 \N +9 \N +10 \N +11 \N +12 2023-02-28 +13 2024-02-29 +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 2023-01-01 +21 \N +22 \N +23 \N +24 2023-01-02 +25 2023-01-01 +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N + diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy index 813c0ca49d94cf..79d508cae87da1 100644 --- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy +++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy @@ -141,4 +141,67 @@ suite("test_from_iso8601_date") { qt_test_87 """ select from_iso8601_date(k1),from_iso8601_date(k2),from_iso8601_date(k3),from_iso8601_date(k11),from_iso8601_date(k22),from_iso8601_date(k33) from tb2 order by k0;""" qt_test_88 """ select from_iso8601_date(nullable(k1)),from_iso8601_date(k2),from_iso8601_date(k3),from_iso8601_date(nullable(k11)),from_iso8601_date(k22),from_iso8601_date(k33) from tb2 order by k0; """ qt_test_89 """ select from_iso8601_date(NULL) from tb2 order by k0; """ + + + sql """ + CREATE TABLE tb3 (id INT, date_str VARCHAR(255)) DISTRIBUTED BY HASH(id) BUCKETS 4 PROPERTIES ("replication_num" = "1"); """ + sql """ + INSERT INTO tb3 (id, date_str) VALUES + (1, '2023-01-01'), + (2, '2023-12-31'), + (3, '2020-02-29'), + (4, '1970-01-01'), + (5, '9999-12-31'), + (6, '1000-01-01'), + (7, '2023-13-01'), + (8, '2023-00-01'), + (9, '2023-01-00'), + (10, '2023-01-32'), + (11, '2023-02-29'), + (12, '2023-02-28'), + (13, '2024-02-29'), + (14, '2024-02-30'), + (15, '2023-01-01T12:00:00'), + (16, '2023-01-01 12:00:00'), + (17, '2023/01/01'), + (18, '01-01-2023'), + (19, '01/01/2023'), + (20, '20230101'), + (21, '2023-01-01Z'), + (22, '2023-01-01+08:00'), + (23, '2023-01-01-08:00'), + (24, '2023-W01-1'), + (25, '2023-001'), + (26, '2023-01-01T12:00:00.000Z'), + (27, '2023-01-01T12:00:00.000+08:00'), + (28, '2023-01-01T12:00:00.000-08:00'), + (29, '2023-01-01T12:00:00.123456Z'), + (30, '2023-01-01T12:00:00.123456+08:00'), + (31, '2023-01-01T12:00:00.123456-08:00'), + (32, '2023-01-01T24:00:00'), + (33, '2023-01-01T00:00:00.000000'), + (34, '2023-01-01T00:00:00.000001'), + (35, '2023-01-01T00:00:00.999999'), + (36, '2023-01-01T23:59:59.999999'), + (37, '2023-01-01T23:59:60'), + (38, '2023-01-01T23:59:59.9999999'), + (39, '2023-01-01T23:59:59.999999999'), + (40, '2023-01-01T23:59:59.999999999Z'), + (41, '2023-01-01T23:59:59.999999999+08:00'), + (42, '2023-01-01T23:59:59.999999999-08:00'), + (43, '2023-01-01T23:59:59.999999999999'), + (44, '2023-01-01T23:59:59.999999999999Z'), + (45, '2023-01-01T23:59:59.999999999999+08:00'), + (46, '2023-01-01T23:59:59.999999999999-08:00'), + (47, '2023-01-01T23:59:59.999999999999999'), + (48, '2023-01-01T23:59:59.999999999999999Z'), + (49, '2023-01-01T23:59:59.999999999999999+08:00'), + (50, '2023-01-01T23:59:59.999999999999999-08:00'); + """ + qt_test_99 """ SELECT id, from_iso8601_date(date_str) AS result FROM tb3 order by id; """ + + + + sql """ drop table tb2 """ + } \ No newline at end of file From 10dd210e2a87447fc849b5f5c873df6a80108b9c Mon Sep 17 00:00:00 2001 From: daidai Date: Wed, 16 Jul 2025 17:06:35 +0800 Subject: [PATCH 2/3] fix --- be/test/vec/function/function_time_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/test/vec/function/function_time_test.cpp b/be/test/vec/function/function_time_test.cpp index 0aa96c62961b8b..61877a2dd16205 100644 --- a/be/test/vec/function/function_time_test.cpp +++ b/be/test/vec/function/function_time_test.cpp @@ -1895,7 +1895,7 @@ TEST(VTimestampFunctionsTest, next_day_test) { TEST(VTimestampFunctionsTest, from_iso8601_date) { std::string func_name = "from_iso8601_date"; - InputTypeSet input_types = {PrimitiveType::TYPE_VARCHAR}; + InputTypeSet input_types = {TypeIndex::String}; DataSet data_set = { {{std::string("2020-01-01")}, std::string("2020-01-01")}, From d275d0e6037c44c41d3e4d1aeefcfb6b3b66b286 Mon Sep 17 00:00:00 2001 From: daidai Date: Thu, 17 Jul 2025 14:17:13 +0800 Subject: [PATCH 3/3] fix --- be/test/vec/function/function_time_test.cpp | 48 ++++++++++----------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/be/test/vec/function/function_time_test.cpp b/be/test/vec/function/function_time_test.cpp index 61877a2dd16205..3d71d70c4a77d4 100644 --- a/be/test/vec/function/function_time_test.cpp +++ b/be/test/vec/function/function_time_test.cpp @@ -1898,50 +1898,50 @@ TEST(VTimestampFunctionsTest, from_iso8601_date) { InputTypeSet input_types = {TypeIndex::String}; DataSet data_set = { - {{std::string("2020-01-01")}, std::string("2020-01-01")}, - {{std::string("2020-01-01")}, std::string("2020-01-01")}, + {{std::string("2020-01-01")}, str_to_date_v2("2020-01-01", "%Y-%m-%d")}, + {{std::string("2020-01-01")}, str_to_date_v2("2020-01-01", "%Y-%m-%d")}, {{std::string("-1")}, Null()}, - {{std::string("2025-07-11")}, std::string("2025-07-11")}, - {{std::string("2024-02-29")}, std::string("2024-02-29")}, + {{std::string("2025-07-11")}, str_to_date_v2("2025-07-11", "%Y-%m-%d")}, + {{std::string("2024-02-29")}, str_to_date_v2("2024-02-29", "%Y-%m-%d")}, {{std::string("2025-02-29")}, Null()}, {{std::string("2025-13-01")}, Null()}, - {{std::string("2020-W10")}, std::string("2020-03-02")}, - {{std::string("2025-W28")}, std::string("2025-07-07")}, - {{std::string("2025-W53")}, std::string("2025-12-29")}, + {{std::string("2020-W10")}, str_to_date_v2("2020-03-02", "%Y-%m-%d")}, + {{std::string("2025-W28")}, str_to_date_v2("2025-07-07", "%Y-%m-%d")}, + {{std::string("2025-W53")}, str_to_date_v2("2025-12-29", "%Y-%m-%d")}, {{std::string("2025-W00")}, Null()}, - {{std::string("2020-123")}, std::string("2020-05-02")}, - {{std::string("2025-192")}, std::string("2025-07-11")}, - {{std::string("2024-366")}, std::string("2024-12-31")}, + {{std::string("2020-123")}, str_to_date_v2("2020-05-02", "%Y-%m-%d")}, + {{std::string("2025-192")}, str_to_date_v2("2025-07-11", "%Y-%m-%d")}, + {{std::string("2024-366")}, str_to_date_v2("2024-12-31", "%Y-%m-%d")}, {{std::string("2025-366")}, Null()}, - {{std::string("2025-000")}, std::string("2024-12-31")}, + {{std::string("2025-000")}, str_to_date_v2("2024-12-31", "%Y-%m-%d")}, {{std::string("2025/07/11")}, Null()}, {{std::string("25-07-11")}, Null()}, {{std::string("2025-7-11")}, Null()}, {{std::string("invalid-date")}, Null()}, {{std::string("2025-07-11T12:34:56")}, Null()}, {{std::string("-1")}, Null()}, - {{std::string("9999-12-31")}, std::string("9999-12-31")}, + {{std::string("9999-12-31")}, str_to_date_v2("9999-12-31", "%Y-%m-%d")}, {{std::string("10000-01-01")}, Null()}, - {{std::string("0001-01-01")}, std::string("0001-01-01")}, - {{std::string("0000-12-31")}, std::string("0000-12-31")}, + {{std::string("0001-01-01")}, str_to_date_v2("0001-01-01", "%Y-%m-%d")}, + {{std::string("0000-12-31")}, str_to_date_v2("0000-12-31", "%Y-%m-%d")}, {{std::string("-0001-01-01")}, Null()}, - {{std::string("2025-01-01")}, std::string("2025-01-01")}, - {{std::string("2025-12-31")}, std::string("2025-12-31")}, + {{std::string("2025-01-01")}, str_to_date_v2("2025-01-01", "%Y-%m-%d")}, + {{std::string("2025-12-31")}, str_to_date_v2("2025-12-31", "%Y-%m-%d")}, {{std::string("2025-00-01")}, Null()}, {{std::string("2025-13-01")}, Null()}, {{std::string("2025--01-01")}, Null()}, - {{std::string("2025-01-31")}, std::string("2025-01-31")}, - {{std::string("2025-04-30")}, std::string("2025-04-30")}, - {{std::string("2025-02-28")}, std::string("2025-02-28")}, - {{std::string("2024-02-29")}, std::string("2024-02-29")}, + {{std::string("2025-01-31")}, str_to_date_v2("2025-01-31", "%Y-%m-%d")}, + {{std::string("2025-04-30")}, str_to_date_v2("2025-04-30", "%Y-%m-%d")}, + {{std::string("2025-02-28")}, str_to_date_v2("2025-02-28", "%Y-%m-%d")}, + {{std::string("2024-02-29")}, str_to_date_v2("2024-02-29", "%Y-%m-%d")}, {{std::string("2025-01-32")}, Null()}, {{std::string("2025-04-31")}, Null()}, {{std::string("2025-02-29")}, Null()}, {{std::string("2025-02-30")}, Null()}, {{std::string("2025-01-00")}, Null()}, {{std::string("2025-01--01")}, Null()}, - {{std::string("2000-02-29")}, std::string("2000-02-29")}, - {{std::string("2024-02-29")}, std::string("2024-02-29")}, + {{std::string("2000-02-29")}, str_to_date_v2("2000-02-29", "%Y-%m-%d")}, + {{std::string("2024-02-29")}, str_to_date_v2("2024-02-29", "%Y-%m-%d")}, {{std::string("1900-02-29")}, Null()}, {{std::string("2100-02-29")}, Null()}, {{std::string("2025-02-29")}, Null()}, @@ -1949,8 +1949,8 @@ TEST(VTimestampFunctionsTest, from_iso8601_date) { {{std::string("2025--07-01")}, Null()}, {{std::string("2025-07--01")}, Null()}, {{std::string("")}, Null()}, - {{std::string("2025")}, std::string("2025-01-01")}, - {{std::string("2025-07")}, std::string("2025-07-01")}, + {{std::string("2025")}, str_to_date_v2("2025-01-01", "%Y-%m-%d")}, + {{std::string("2025-07")}, str_to_date_v2("2025-07-01", "%Y-%m-%d")}, {{std::string("99999-01-01")}, Null()}, {{std::string("2025-123-01")}, Null()}, {{std::string("2025-01-123")}, Null()},