From b9f8377bb68d15bda8f4a750ce1d4234d59bd389 Mon Sep 17 00:00:00 2001 From: amorynan Date: Wed, 20 Mar 2024 15:57:58 +0800 Subject: [PATCH 1/2] fix scale --- be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp index 10f5f0e79f7146..334252d1898c39 100644 --- a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp @@ -79,8 +79,8 @@ Status DataTypeDateTimeV2SerDe::deserialize_one_cell_from_json(IColumn& column, } } else if (ReadBuffer rb(slice.data, slice.size); - !read_datetime_v2_text_impl(val, rb)) { - return Status::InvalidDataFormat("parse date fail, string: '{}'", + !read_datetime_v2_text_impl(val, rb, scale)) { + return Status::InvalidArgument("parse date fail, string: '{}'", std::string(rb.position(), rb.count()).c_str()); } column_data.insert_value(val); From c941ab4b4d3299dd2c04f7596077dddc3b218c01 Mon Sep 17 00:00:00 2001 From: amorynan Date: Thu, 21 Mar 2024 11:42:37 +0800 Subject: [PATCH 2/2] fixed --- .../serde/data_type_datetimev2_serde.cpp | 2 +- .../test_array_with_scale_type.csv | 2 + .../test_array_with_scale_type.out | 74 +++++++++++++++++++ .../test_array_with_scale_type.groovy | 22 ++++++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.csv diff --git a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp index 334252d1898c39..7dc7dbf17c65dc 100644 --- a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp @@ -81,7 +81,7 @@ Status DataTypeDateTimeV2SerDe::deserialize_one_cell_from_json(IColumn& column, } else if (ReadBuffer rb(slice.data, slice.size); !read_datetime_v2_text_impl(val, rb, scale)) { return Status::InvalidArgument("parse date fail, string: '{}'", - std::string(rb.position(), rb.count()).c_str()); + std::string(rb.position(), rb.count()).c_str()); } column_data.insert_value(val); return Status::OK(); diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.csv b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.csv new file mode 100644 index 00000000000000..a14f2b3fd6eb42 --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.csv @@ -0,0 +1,2 @@ +3|"2022-12-01 22:23:24.999999"|22.6789|33.6789|["2022-12-01 22:23:24.999999","2022-12-01 23:23:24.999999"]|[22.6789,33.6789]|[22.6789,33.6789] +4|"2022-12-02 22:23:24.999999"|23.6789|34.6789|["2022-12-02 22:23:24.999999","2022-12-02 23:23:24.999999"]|[23.6789,34.6789]|[22.6789,34.6789] diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out index 8758907842bb2b..278e084c87d635 100644 --- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_with_scale_type.out @@ -6,6 +6,8 @@ 2022-12-02T22:23:24.999 -- !select -- +2022-12-01T22:23:25 +2022-12-02T22:23:25 2022-12-01T22:23:24.999 2022-12-02T22:23:24.999 @@ -16,6 +18,8 @@ 2022-12-02T22:23:24.999 -- !select -- +2022-12-01T23:23:25 +2022-12-02T23:23:25 2022-12-01T23:23:24.999 2022-12-02T23:23:24.999 @@ -31,10 +35,14 @@ -- !select -- 22.679 23.679 +22.679 +23.679 -- !select -- 22.679 22.679 +22.679 +22.679 -- !select -- 23 @@ -48,18 +56,26 @@ -- !select -- 33.679 34.679 +33.679 +34.679 -- !select -- 33.679 34.679 +33.679 +34.679 -- !select -- [22.679] [23.679] +[22.679] +[23.679] -- !select -- [24.990, 25.990] [24.990, 25.990] +[24.990, 25.990] +[24.990, 25.990] -- !select -- [24.990, 25.990] @@ -67,21 +83,29 @@ -- !select -- [33.679] [34.679] +[33.679] +[34.679] -- !select -- [24.990, 25.990] [24.990, 25.990] +[24.990, 25.990] +[24.990, 25.990] -- !select -- [24.990, 25.990] -- !select -- +[null] +[null] ["2022-12-01 22:23:24.999"] ["2022-12-02 22:23:24.999"] -- !select -- ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] -- !select -- [2022-12-02 22:23:24.999, 2022-12-02 22:23:23.997] @@ -89,98 +113,146 @@ -- !select -- [] [] +[] +[] -- !select -- +["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] +["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] -- !select -- \N \N +\N +\N -- !select -- [22.679] [] +[22.679] +[] -- !select -- [22.679, 33.679] [23.679, 34.679] +[22.679, 33.679] +[23.679, 34.679] -- !select -- \N \N +\N +\N -- !select -- ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] -- !select -- +["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] +["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] -- !select -- +["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000", "2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000", "2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999", "2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999", "2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] -- !select -- [22.679, 33.679, 22.679, 33.679, 22.679, 33.679] [23.679, 34.679, 23.679, 34.679, 23.679, 34.679] +[22.679, 33.679, 22.679, 33.679, 22.679, 33.679] +[23.679, 34.679, 23.679, 34.679, 23.679, 34.679] -- !select -- +[{"col": 22.679, "col": 22.679, "col": "2022-12-01 22:23:25.000", "col": 22.679}, {"col": 33.679, "col": 33.679, "col": "2022-12-01 23:23:25.000", "col": 33.679}] +[{"col": 23.679, "col": 23.679, "col": "2022-12-02 22:23:25.000", "col": 23.679}, {"col": 34.679, "col": 34.679, "col": "2022-12-02 23:23:25.000", "col": 34.679}] [{"col": 22.679, "col": 22.679, "col": "2022-12-01 22:23:24.999", "col": 22.679}, {"col": 33.679, "col": 33.679, "col": "2022-12-01 23:23:24.999", "col": 33.679}] [{"col": 23.679, "col": 23.679, "col": "2022-12-02 22:23:24.999", "col": 23.679}, {"col": 34.679, "col": 34.679, "col": "2022-12-02 23:23:24.999", "col": 34.679}] -- !select -- [{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 22:23:23.997"}] [{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 22:23:23.997"}] +[{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 22:23:23.997"}] +[{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 22:23:23.997"}] -- !select -- +[{"col": "2022-12-01 22:23:25.000"}, {"col": "2022-12-01 23:23:25.000"}] +[{"col": "2022-12-02 22:23:25.000"}, {"col": "2022-12-02 23:23:25.000"}] [{"col": "2022-12-01 22:23:24.999"}, {"col": "2022-12-01 23:23:24.999"}] [{"col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 23:23:24.999"}] -- !select -- +[{"col": "2022-12-01 22:23:25.000", "col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-01 23:23:25.000", "col": "2022-12-02 22:23:23.997"}] +[{"col": "2022-12-02 22:23:25.000", "col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 23:23:25.000", "col": "2022-12-02 22:23:23.997"}] [{"col": "2022-12-01 22:23:24.999", "col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-01 23:23:24.999", "col": "2022-12-02 22:23:23.997"}] [{"col": "2022-12-02 22:23:24.999", "col": "2022-12-02 22:23:24.999"}, {"col": "2022-12-02 23:23:24.999", "col": "2022-12-02 22:23:23.997"}] -- !select -- ["2022-12-02 22:23:23.997", "2022-12-02 22:23:24.999"] ["2022-12-02 22:23:23.997", "2022-12-02 22:23:24.999"] +["2022-12-02 22:23:23.997", "2022-12-02 22:23:24.999"] +["2022-12-02 22:23:23.997", "2022-12-02 22:23:24.999"] -- !select -- +["2023-03-08 23:23:23.997", "2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] +["2023-03-08 23:23:23.997", "2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2023-03-08 23:23:23.997", "2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2023-03-08 23:23:23.997", "2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] -- !select -- +\N ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] [null, "2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] +\N ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] [null, "2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] 2022-12-01T22:23:24.999 ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2022-12-01 22:23:24.999", "2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] 2022-12-02T22:23:24.999 ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] ["2022-12-02 22:23:24.999", "2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] -- !select -- [25.990, 22.679, 33.679] [25.990, 23.679, 34.679] +[25.990, 22.679, 33.679] +[25.990, 23.679, 34.679] -- !select -- 22.679 [22.679, 33.679] [22.679, 22.679, 33.679] 23.679 [23.679, 34.679] [23.679, 23.679, 34.679] +22.679 [22.679, 33.679] [22.679, 22.679, 33.679] +23.679 [23.679, 34.679] [23.679, 23.679, 34.679] -- !select -- ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] ["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] +["2022-12-02 22:23:24.999", "2022-12-02 22:23:23.997"] -- !select -- +["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000", "2023-03-08 23:23:23.997"] +["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000", "2023-03-08 23:23:23.997"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999", "2023-03-08 23:23:23.997"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999", "2023-03-08 23:23:23.997"] -- !select -- +\N ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000"] ["2022-12-01 22:23:25.000", "2022-12-01 23:23:25.000", null] +\N ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000"] ["2022-12-02 22:23:25.000", "2022-12-02 23:23:25.000", null] 2022-12-01T22:23:24.999 ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999"] ["2022-12-01 22:23:24.999", "2022-12-01 23:23:24.999", "2022-12-01 22:23:24.999"] 2022-12-02T22:23:24.999 ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999"] ["2022-12-02 22:23:24.999", "2022-12-02 23:23:24.999", "2022-12-02 22:23:24.999"] -- !select -- [22.679, 33.679, 25.990] [23.679, 34.679, 25.990] +[22.679, 33.679, 25.990] +[23.679, 34.679, 25.990] -- !select -- 22.679 [22.679, 33.679] [22.679, 33.679, 22.679] 23.679 [23.679, 34.679] [23.679, 34.679, 23.679] +22.679 [22.679, 33.679] [22.679, 33.679, 22.679] +23.679 [23.679, 34.679] [23.679, 34.679, 23.679] -- !select -- [23, 11] @@ -194,4 +266,6 @@ -- !select -- [22.679, 56.358] [23.679, 58.358] +[22.679, 56.358] +[23.679, 58.358] diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_with_scale_type.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_with_scale_type.groovy index 53a37100f0ea1f..96a616bcac2881 100644 --- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_with_scale_type.groovy +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_with_scale_type.groovy @@ -36,6 +36,28 @@ suite("test_array_with_scale_type") { "storage_format" = "V2" ) """ + // load with same insert into data + streamLoad { + table "${tableName}" + + set 'column_separator', '|' + + file 'test_array_with_scale_type.csv' + time 10000 // limit inflight 10s + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(2, json.NumberTotalRows) + assertEquals(2, json.NumberLoadedRows) + assertEquals(0, json.NumberFilteredRows) + assertEquals(0, json.NumberUnselectedRows) + } + } sql """INSERT INTO ${tableName} values (1,"2022-12-01 22:23:24.999999",22.6789,33.6789,["2022-12-01 22:23:24.999999","2022-12-01 23:23:24.999999"],[22.6789,33.6789],[22.6789,33.6789]),