From 2a3c0fa4ce60a61e0174bc069915482b7afac4ab Mon Sep 17 00:00:00 2001 From: qiye Date: Wed, 8 Mar 2023 21:51:17 +0800 Subject: [PATCH 1/9] [fix](DOE) Fix esquery not working (#17566) Function esquery does not work because there is a problem parsing the first parameter type. The first parameter, which is SlotRef, will be cast to CastExpr. This will cause error while generating ES DSL. Add more types to adapt esquery function. --- gensrc/script/doris_builtins_functions.py | 41 +++++++--- regression-test/data/es_p0/test_es_query.out | 75 ++++++++++--------- .../suites/es_p0/test_es_query.groovy | 9 ++- 3 files changed, 79 insertions(+), 46 deletions(-) diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 63dd7b3d011ba1..33f5dfb771dba0 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -420,7 +420,7 @@ [['array_remove'], 'ARRAY_DATETIMEV2', ['ARRAY_DATETIMEV2', 'DATETIMEV2'], '', '', '', 'vec', ''], [['array_remove'], 'ARRAY_DATEV2', ['ARRAY_DATEV2', 'DATEV2'], '', '', '', 'vec', ''], [['array_remove'], 'ARRAY_VARCHAR', ['ARRAY_VARCHAR', 'VARCHAR'], '', '', '', 'vec', ''], - [['array_remove'], 'ARRAY_STRING', ['ARRAY_STRING', 'STRING'], '', '', '', 'vec', ''], + [['array_remove'], 'ARRAY_STRING', ['ARRAY_STRING', 'STRING'], '', '', '', 'vec', ''], [['array_union'], 'ARRAY_BOOLEAN', ['ARRAY_BOOLEAN', 'ARRAY_BOOLEAN'], '', '', '', 'vec', ''], [['array_union'], 'ARRAY_TINYINT', ['ARRAY_TINYINT', 'ARRAY_TINYINT'], '', '', '', 'vec', ''], @@ -659,7 +659,7 @@ '_ZN5doris18TimestampFunctions8last_dayEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', '', '', 'vec', 'ALWAYS_NULLABLE'], [['last_day'], 'DATEV2', ['DATETIMEV2'], - '_ZN5doris18TimestampFunctions8last_dayEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', + '_ZN5doris18TimestampFunctions8last_dayEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', '', '', 'vec', ''], [['last_day'], 'DATEV2', ['DATEV2'], '_ZN5doris18TimestampFunctions8last_dayEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', @@ -677,10 +677,10 @@ '', '', 'vec', 'ALWAYS_NULLABLE'], [['date_trunc'], 'DATETIME', ['DATETIME', 'VARCHAR'], - '', '', '', 'vec', 'ALWAYS_NULLABLE'], + '', '', '', 'vec', 'ALWAYS_NULLABLE'], [['date_trunc'], 'DATETIMEV2', ['DATETIMEV2', 'VARCHAR'], - '', '', '', 'vec', 'ALWAYS_NULLABLE'], + '', '', '', 'vec', 'ALWAYS_NULLABLE'], [['year'], 'INT', ['DATETIME'], '_ZN5doris18TimestampFunctions4yearEPN9doris_udf15FunctionContextERKNS1_11DateTimeValE', @@ -2154,9 +2154,32 @@ [['coalesce'], 'VARCHAR', ['VARCHAR', '...'], '', '', '', 'vec', 'CUSTOM'], [['coalesce'], 'STRING', ['STRING', '...'], '', '', '', 'vec', 'CUSTOM'], - [['esquery'], 'BOOLEAN', ['VARCHAR', 'VARCHAR'], - '_ZN5doris11ESFunctions5matchEPN' - '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + # esquery's first arg is suitable for all types + [['esquery'], 'BOOLEAN', ['BOOLEAN', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['TINYINT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['SMALLINT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['INT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['BIGINT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['LARGEINT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['FLOAT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['DOUBLE', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['DATE', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['DATETIME', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['CHAR', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['VARCHAR', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['JSONB', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['DECIMALV2', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['DECIMAL32', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['DECIMAL64', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['DECIMAL128', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['TIME', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['DATEV2', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['DATETIMEV2', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['TIMEV2', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['ARRAY', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['MAP', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['STRING', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['VARIANT', 'VARCHAR'], ''], # String builtin functions [['substr', 'substring'], 'VARCHAR', ['VARCHAR', 'INT'], @@ -2261,10 +2284,10 @@ '', '', 'vec', ''], [['elt'], 'VARCHAR', ['INT', 'VARCHAR', '...'], '_ZN5doris15StringFunctions3eltEPN9doris_udf15FunctionContextERKNS1_6IntValEiPKNS1_9StringValE', - '', '', 'vec', ''], + '', '', 'vec', ''], [['elt'], 'STRING', ['INT', 'STRING', '...'], '_ZN5doris15StringFunctions3eltEPN9doris_udf15FunctionContextERKNS1_6IntValEiPKNS1_9StringValE', - '', '', 'vec', ''], + '', '', 'vec', ''], [['replace'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'], '_ZN5doris15StringFunctions7replaceEPN9doris_udf15FunctionContextERKNS1_9StringValES6_S6_', '', '', 'vec', ''], diff --git a/regression-test/data/es_p0/test_es_query.out b/regression-test/data/es_p0/test_es_query.out index 72c21f8acf5109..cd691cc049c7d1 100644 --- a/regression-test/data/es_p0/test_es_query.out +++ b/regression-test/data/es_p0/test_es_query.out @@ -1,67 +1,72 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !sql51 -- +[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] .14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] + -- !sql52 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] .14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] --- !sql51 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +-- !sql53 -- +[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] .14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] + +-- !sql54 -- +[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] .14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -- !sql62 -- -['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00] ['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 -- !sql63 -- -['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string2 [1, 2, 3, 4] 2022-08-08 text2 [2020-01-01, 2020-01-02] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string3 [1, 2, 3, 4] 2022-08-08 text3_4*5 [2020-01-01, 2020-01-02] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -- !sql64 -- -['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string2 [1, 2, 3, 4] 2022-08-08 text2 [2020-01-01, 2020-01-02] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00] ['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string2 text2 4.0 2022-08-08T00:00 -- !sql65 -- -true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01T00:00 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01T00:00 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01T00:00 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} -- !sql66 -- -true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01T00:00 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01T00:00 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01T00:00 2020-01-01 12:00:00 a d 192.168.0.1 {"name":"Andy","age":18} + +-- !sql67 -- +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00] ['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 -- !sql72 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08 2022-08-08 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 1659931810000 -- !sql73 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08 2022-08-08 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string2 [1, 2, 3, 4] 2022-08-08 2022-08-09 text2 [2020-01-01, 2020-01-02] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-09 2022-08-09 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string3 [1, 2, 3, 4] 2022-08-08 2022-08-10 text3_4*5 [2020-01-01, 2020-01-02] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-09 2022-08-10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -- !sql74 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string2 [1, 2, 3, 4] 2022-08-08 2022-08-09 text2 [2020-01-01, 2020-01-02] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-09 2022-08-09 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string2 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09 12:10:10 -- !sql75 -- -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} -- !sql76 -- -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} + +-- !sql77 -- +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 1659931810000 -- !sql81 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08 2022-08-08 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 1659931810000 -- !sql82 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08 2022-08-08 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string2 [1, 2, 3, 4] 2022-08-08 2022-08-09 text2 [2020-01-01, 2020-01-02] 4.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-09 2022-08-09 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string3 [1, 2, 3, 4] 2022-08-08 2022-08-10 text3_4*5 [2020-01-01, 2020-01-02] 5.0 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-09 2022-08-10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -- !sql83 -- -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} -- !sql84 -- -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} -true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +-- !sql85 -- +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 1659931810000 diff --git a/regression-test/suites/es_p0/test_es_query.groovy b/regression-test/suites/es_p0/test_es_query.groovy index db503508a10d84..da7b0855658b4f 100644 --- a/regression-test/suites/es_p0/test_es_query.groovy +++ b/regression-test/suites/es_p0/test_es_query.groovy @@ -94,7 +94,8 @@ suite("test_es_query", "p0") { "http_ssl_enabled"="false" ); """ - order_qt_sql52 """select * from test_v1 where test2='text#1'""" + order_qt_sql51 """select * from test_v1 where test2='text#1'""" + order_qt_sql52 """select * from test_v1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" sql """ CREATE TABLE `test_v2` ( @@ -131,7 +132,8 @@ suite("test_es_query", "p0") { "http_ssl_enabled"="false" ); """ - order_qt_sql51 """select * from test_v2 where test2='text#1'""" + order_qt_sql53 """select * from test_v2 where test2='text#1'""" + order_qt_sql54 """select * from test_v2 where esquery(test2, '{"match":{"test2":"text#1"}}')""" sql """create catalog if not exists es6 with resource es6_resource;""" @@ -144,6 +146,7 @@ suite("test_es_query", "p0") { order_qt_sql64 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" order_qt_sql65 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test1""" order_qt_sql66 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test2_20220808""" + order_qt_sql67 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" sql """switch es7""" // order_qt_sql71 """show tables""" order_qt_sql72 """select * from test1 where test2='text#1'""" @@ -151,11 +154,13 @@ suite("test_es_query", "p0") { order_qt_sql74 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" order_qt_sql75 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test1""" order_qt_sql76 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test2""" + order_qt_sql77 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" sql """switch es8""" order_qt_sql81 """select * from test1 where test2='text#1'""" order_qt_sql82 """select * from test2_20220808 where test4='2022-08-08'""" order_qt_sql83 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test1""" order_qt_sql84 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test2""" + order_qt_sql85 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" sql """drop catalog if exists es6;""" From 8f04edbd539846a4c301efff73532b4564a6782d Mon Sep 17 00:00:00 2001 From: huangzhaowei Date: Fri, 10 Mar 2023 09:59:20 +0800 Subject: [PATCH 2/9] [Enhance](DOE)Support parse default es iso datetime string (#17412) * support parse default es iso datetime string --- be/src/exec/es/es_scroll_parser.cpp | 57 ++++++++++++++++--- .../elasticsearch/scripts/data/data1.json | 1 + .../elasticsearch/scripts/data/data2.json | 1 + .../elasticsearch/scripts/data/data3.json | 5 +- .../elasticsearch/scripts/data/data4.json | 29 ++++++++++ .../elasticsearch/scripts/es_init.sh | 21 +++++-- .../scripts/index/es7_test1.json | 3 + .../scripts/index/es7_test2.json | 3 + .../doris/external/elasticsearch/EsUtil.java | 7 ++- regression-test/data/es_p0/test_es_query.out | 46 ++++++++++++--- .../suites/es_p0/test_es_query.groovy | 11 ++-- 11 files changed, 151 insertions(+), 33 deletions(-) create mode 100755 docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4.json diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index f0b73c0323a34d..6a0587e48c72cc 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -17,6 +17,7 @@ #include "exec/es/es_scroll_parser.h" +#include #include #include @@ -138,14 +139,6 @@ static const std::string INVALID_NULL_VALUE = return Status::RuntimeError(ss.str()); \ } while (false) -#define PARSE_DATE(dt_val, col, type, is_date_str) \ - if ((is_date_str && \ - !dt_val.from_date_str(static_cast(col.GetString()).c_str(), \ - col.GetStringLength())) || \ - (!is_date_str && !dt_val.from_unixtime(col.GetInt64() / 1000, "+08:00"))) { \ - RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type); \ - } - template static Status get_int_value(const rapidjson::Value& col, PrimitiveType type, void* slot, bool pure_doc_value) { @@ -184,8 +177,54 @@ static Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type RT* slot) { constexpr bool is_datetime_v1 = std::is_same_v; T dt_val; - PARSE_DATE(dt_val, col, type, is_date_str) + if (is_date_str) { + const std::string str_date = col.GetString(); + int str_length = col.GetStringLength(); + bool success = false; + // YYYY-MM-DDTHH:MM:SSZ or YYYY-MM-DDTHH:MM:SS+08:00 or 2022-08-08T12:10:10.000Z + if (str_length > 19) { + std::chrono::system_clock::time_point tp; + const bool ok = + cctz::parse("%Y-%m-%dT%H:%M:%E*S%Ez", str_date, cctz::utc_time_zone(), &tp); + if (ok) { + success = dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), + cctz::local_time_zone().name()); + } + } else if (str_length == 19) { + // YYYY-MM-DDTHH:MM:SS + if (*(str_date.c_str() + 10) == 'T') { + std::chrono::system_clock::time_point tp; + const bool ok = + cctz::parse("%Y-%m-%dT%H:%M:%S", str_date, cctz::utc_time_zone(), &tp); + if (ok) { + success = dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), + cctz::local_time_zone().name()); + } + } else { + // YYYY-MM-DD HH:MM:SS + success = dt_val.from_date_str(str_date.c_str(), str_length); + } + } else if (str_length == 13) { + // string long like "1677895728000" + int64_t time_long = std::atol(str_date.c_str()); + if (time_long > 0) { + success = dt_val.from_unixtime(time_long / 1000, cctz::local_time_zone().name()); + } + } else { + // YYYY-MM-DD or others + success = dt_val.from_date_str(str_date.c_str(), str_length); + } + + if (!success) { + RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type); + } + + } else { + if (!dt_val.from_unixtime(col.GetInt64() / 1000, cctz::local_time_zone().name())) { + RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type); + } + } if constexpr (is_datetime_v1) { if (type == TYPE_DATE) { dt_val.cast_to_date(); diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1.json index 12bcbb0d5c8072..030246786820ed 100755 --- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1.json +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data1.json @@ -6,6 +6,7 @@ "test5": "2022-08-08 12:10:10", "test6": 1659931810000, "test7": 1659931810000, + "test8": "2022-08-08T12:10:10Z", "c_bool": [true, false, true, true], "c_byte": [1, -2, -3, 4], "c_short": [128, 129, -129, -130], diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json index 3b9ebdc6f2f369..c6f7d3e2f6ada6 100755 --- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json @@ -6,6 +6,7 @@ "test5": "2022-08-09 12:10:10", "test6": 1660018210000, "test7": "2022-08-09 12:10:10", + "test8": 1660018210000, "c_bool": [true, false, true, true], "c_byte": [1, -2, -3, 4], "c_short": [128, 129, -129, -130], diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json index 9c10c2cf2a9ac7..73631d826185c7 100755 --- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json @@ -4,8 +4,9 @@ "test3": 5.0, "test4": "2022-08-08", "test5": "2022-08-10 12:10:10", - "test6": 1660018210000, - "test7": "2022-08-10 12:10:10", + "test6": 1660104610000, + "test7": 1660104610000, + "test8": "2022-08-10T12:10:10", "c_bool": [true, false, true, true], "c_byte": [1, -2, -3, 4], "c_short": [128, 129, -129, -130], diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4.json new file mode 100755 index 00000000000000..954ca884d61567 --- /dev/null +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data4.json @@ -0,0 +1,29 @@ +{ + "test1": "string4", + "test2": "text3_4*5", + "test3": 6.0, + "test4": "2022-08-08", + "test5": "2022-08-11 12:10:10", + "test6": 1660191010000, + "test7": "2022-08-11 12:10:10", + "test8": "2022-08-11T12:10:10+09:00", + "c_bool": [true, false, true, true], + "c_byte": [1, -2, -3, 4], + "c_short": [128, 129, -129, -130], + "c_integer": [32768, 32769, -32769, -32770], + "c_long": [-1, 0, 1, 2], + "c_unsigned_long": [0, 1, 2, 3], + "c_float": [1.0, 1.1, 1.2, 1.3], + "c_half_float": [1, 2, 3, 4], + "c_double": [1, 2, 3, 4], + "c_scaled_float": [1, 2, 3, 4], + "c_date": ["2020-01-01", "2020-01-02"], + "c_datetime": ["2020-01-01 12:00:00", "2020-01-02 13:01:01"], + "c_keyword": ["a", "b", "c"], + "c_text": ["d", "e", "f"], + "c_ip": ["192.168.0.1", "127.0.0.1"], + "c_person": [ + {"name": "Andy", "age": 18}, + {"name": "Tim", "age": 28} + ] +} \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh b/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh index 13947a064c0792..8ddb125e327ff2 100755 --- a/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/es_init.sh @@ -20,12 +20,13 @@ # create index test1 # shellcheck disable=SC2154 curl "http://${ES_6_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es6_test1.json" -# create index test2 +# create index test2_20220808 curl "http://${ES_6_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es6_test2.json' -# put data +# put data for test1 curl "http://${ES_6_HOST}:9200/test1/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json' curl "http://${ES_6_HOST}:9200/test1/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json' curl "http://${ES_6_HOST}:9200/test1/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3_es6.json' +# put data for test2_20220808 curl "http://${ES_6_HOST}:9200/test2_20220808/doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1_es6.json' curl "http://${ES_6_HOST}:9200/test2_20220808/doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2_es6.json' curl "http://${ES_6_HOST}:9200/test2_20220808/doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3_es6.json' @@ -36,15 +37,19 @@ curl "http://${ES_6_HOST}:9200/test2_20220808/doc/_mapping" -H "Content-Type:app # es7 # create index test1 curl "http://${ES_7_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es7_test1.json" -# create index test2 +# create index test2_20220808 curl "http://${ES_7_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json' -# put data +# put data for tese1 curl "http://${ES_7_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json' curl "http://${ES_7_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' curl "http://${ES_7_HOST}:9200/test1/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json' +curl "http://${ES_7_HOST}:9200/test1/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json' +# put data for test2_20220808 curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json' curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json' +curl "http://${ES_7_HOST}:9200/test2_20220808/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json' + # put _meta for array curl "http://${ES_7_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" curl "http://${ES_7_HOST}:9200/test2_20220808/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" @@ -52,15 +57,19 @@ curl "http://${ES_7_HOST}:9200/test2_20220808/_mapping" -H "Content-Type:applica # es8 # create index test1 curl "http://${ES_8_HOST}:9200/test1" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/es7_test1.json" -# create index test2 +# create index test2_20220808 curl "http://${ES_8_HOST}:9200/test2_20220808" -H "Content-Type:application/json" -X PUT -d '@/mnt/scripts/index/es7_test2.json' -# put data +# put data for tese1 curl "http://${ES_8_HOST}:9200/test1/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json' curl "http://${ES_8_HOST}:9200/test1/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' curl "http://${ES_8_HOST}:9200/test1/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json' +curl "http://${ES_8_HOST}:9200/test1/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json' +# put data for test2_20220808 curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/1" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data1.json' curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/2" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data2.json' curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/3" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data3.json' +curl "http://${ES_8_HOST}:9200/test2_20220808/_doc/4" -H "Content-Type:application/json" -X POST -d '@/mnt/scripts/data/data4.json' + # put _meta for array curl "http://${ES_8_HOST}:9200/test1/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" curl "http://${ES_8_HOST}:9200/test2_20220808/_mapping" -H "Content-Type:application/json" -X PUT -d "@/mnt/scripts/index/array_meta.json" diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json index ebc4227a203b7b..4f49a1db8b4ad8 100755 --- a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json @@ -35,6 +35,9 @@ "type": "date", "format": "yyyy-MM-dd HH:mm:ss || epoch_millis" }, + "test8": { + "type": "date" + }, "c_bool": { "type": "boolean" }, diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test2.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test2.json index 00143131d001a3..166331f5ccfdc1 100755 --- a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test2.json +++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test2.json @@ -38,6 +38,9 @@ "type": "date", "format": "yyyy-MM-dd HH:mm:ss || epoch_millis" }, + "test8": { + "type": "date" + }, "c_bool": { "type": "boolean" }, diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java index ef26eaf0e43365..f2980133528e5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java @@ -303,12 +303,13 @@ private static Type parseEsDateType(Column column, ObjectNode field) { boolean bigIntFlag = false; for (String format : formats) { // pre-check format - if (!ALLOW_DATE_FORMATS.contains(format)) { + String trimFormat = format.trim(); + if (!ALLOW_DATE_FORMATS.contains(trimFormat)) { column.setComment( - "Elasticsearch type is date, format is " + format + " not support, use String type"); + "Elasticsearch type is date, format is " + trimFormat + " not support, use String type"); return ScalarType.createStringType(); } - switch (format) { + switch (trimFormat) { case "yyyy-MM-dd HH:mm:ss": dateTimeFlag = true; break; diff --git a/regression-test/data/es_p0/test_es_query.out b/regression-test/data/es_p0/test_es_query.out index cd691cc049c7d1..57ddfafd02f643 100644 --- a/regression-test/data/es_p0/test_es_query.out +++ b/regression-test/data/es_p0/test_es_query.out @@ -1,20 +1,35 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql51 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] .14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -- !sql52 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] .14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -- !sql53 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] .14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +2022-08-08 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T04:10:10 2022-08-08T20:10:10 +2022-08-08 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 +2022-08-08 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T04:10:10 2022-08-10T20:10:10 +2022-08-08 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 + +-- !sql53 -- +[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] -- !sql54 -- -[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] .14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] +[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] + +-- !sql55 -- +2022-08-08 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T04:10:10 2022-08-08T20:10:10 +2022-08-08 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 2022-08-09T12:10:10 +2022-08-08 2022-08-10T12:10:10 2022-08-10T12:10:10 2022-08-10T04:10:10 2022-08-10T20:10:10 +2022-08-08 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T12:10:10 2022-08-11T11:10:10 -- !sql62 -- [1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00] ['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 -- !sql63 -- +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00] ['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00] ['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string2 text2 4.0 2022-08-08T00:00 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00] ['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string3 text3_4*5 5.0 2022-08-08T00:00 -- !sql64 -- [1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00] ['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string2 text2 4.0 2022-08-08T00:00 @@ -33,40 +48,53 @@ true 1 128 32768 -1 0 1.0 1 1 1 2020-01-01T00:00 2020-01-01 12:00:00 a d 192.168 [1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00] ['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 -- !sql72 -- -[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 1659931810000 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 -- !sql73 -- +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string2 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string3 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string4 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 -- !sql74 -- -[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string2 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09 12:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string2 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 -- !sql75 -- true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} -- !sql76 -- true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} -- !sql77 -- -[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 1659931810000 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 -- !sql81 -- -[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 1659931810000 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 -- !sql82 -- +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string2 text2 4.0 2022-08-08T00:00 2022-08-09T12:10:10 1660018210000 2022-08-09T12:10:10 2022-08-09T12:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string3 text3_4*5 5.0 2022-08-08T00:00 2022-08-10T12:10:10 1660104610000 2022-08-10T12:10:10 2022-08-10T20:10:10 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string4 text3_4*5 6.0 2022-08-08T00:00 2022-08-11T12:10:10 1660191010000 2022-08-11T12:10:10 2022-08-11T11:10:10 -- !sql83 -- true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} -- !sql84 -- true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} +true 1 128 32768 -1 0 1.0 1.0 1.0 1.0 2020-01-01 2020-01-01T12:00 a d 192.168.0.1 {"name":"Andy","age":18} -- !sql85 -- -[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 1659931810000 +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 + diff --git a/regression-test/suites/es_p0/test_es_query.groovy b/regression-test/suites/es_p0/test_es_query.groovy index da7b0855658b4f..67306fb788964b 100644 --- a/regression-test/suites/es_p0/test_es_query.groovy +++ b/regression-test/suites/es_p0/test_es_query.groovy @@ -81,6 +81,7 @@ suite("test_es_query", "p0") { `c_person` array NULL, `test6` datetime NULL, `test7` datetime NULL, + `test8` datetime NULL, `c_byte` array NULL, `c_bool` array NULL, `c_integer` array NULL @@ -96,6 +97,7 @@ suite("test_es_query", "p0") { """ order_qt_sql51 """select * from test_v1 where test2='text#1'""" order_qt_sql52 """select * from test_v1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" + order_qt_sql53 """select test4,test5,test6,test7,test8 from test_v1 order by test8""" sql """ CREATE TABLE `test_v2` ( @@ -119,6 +121,7 @@ suite("test_es_query", "p0") { `c_person` array NULL, `test6` datetimev2 NULL, `test7` datetimev2 NULL, + `test8` datetimev2 NULL, `c_byte` array NULL, `c_bool` array NULL, `c_integer` array NULL @@ -134,7 +137,7 @@ suite("test_es_query", "p0") { """ order_qt_sql53 """select * from test_v2 where test2='text#1'""" order_qt_sql54 """select * from test_v2 where esquery(test2, '{"match":{"test2":"text#1"}}')""" - + order_qt_sql55 """select test4,test5,test6,test7,test8 from test_v2 order by test8""" sql """create catalog if not exists es6 with resource es6_resource;""" sql """create catalog if not exists es7 with resource es7_resource;""" @@ -142,7 +145,7 @@ suite("test_es_query", "p0") { sql """switch es6""" // order_qt_sql61 """show tables""" order_qt_sql62 """select * from test1 where test2='text#1'""" - order_qt_sql63 """select * from test2_20220808 where test4='2022-08-08'""" + order_qt_sql63 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" order_qt_sql64 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" order_qt_sql65 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test1""" order_qt_sql66 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test2_20220808""" @@ -150,14 +153,14 @@ suite("test_es_query", "p0") { sql """switch es7""" // order_qt_sql71 """show tables""" order_qt_sql72 """select * from test1 where test2='text#1'""" - order_qt_sql73 """select * from test2_20220808 where test4='2022-08-08'""" + order_qt_sql73 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" order_qt_sql74 """select * from test2_20220808 where substring(test2, 2) = 'ext2'""" order_qt_sql75 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test1""" order_qt_sql76 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test2""" order_qt_sql77 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" sql """switch es8""" order_qt_sql81 """select * from test1 where test2='text#1'""" - order_qt_sql82 """select * from test2_20220808 where test4='2022-08-08'""" + order_qt_sql82 """select * from test2_20220808 where test4 >= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" order_qt_sql83 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test1""" order_qt_sql84 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test2""" order_qt_sql85 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" From 66f7313f02f14c503ed6bffecb399183cb7fb8db Mon Sep 17 00:00:00 2001 From: huangzhaowei Date: Tue, 21 Mar 2023 10:08:20 +0800 Subject: [PATCH 3/9] [BugFix](DOE) Add http prefix when it's not set in hosts properties. (#17745) * Add http prefix when it's not set in hosts properties --- .../org/apache/doris/catalog/EsResource.java | 9 + .../org/apache/doris/catalog/EsTable.java | 8 +- .../doris/datasource/EsExternalCatalog.java | 6 +- .../data/es_p0/test_es_query_no_http_url.out | 16 ++ .../es_p0/test_es_query_no_http_url.groovy | 160 ++++++++++++++++++ 5 files changed, 196 insertions(+), 3 deletions(-) create mode 100644 regression-test/data/es_p0/test_es_query_no_http_url.out create mode 100644 regression-test/suites/es_p0/test_es_query_no_http_url.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java index 91cb4529ad73c7..84f51cb820b7d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java @@ -136,6 +136,15 @@ public static void valid(Map properties, boolean isAlter) throws } } + public static void fillUrlsWithSchema(String[] urls, boolean isSslEnabled) { + for (int i = 0; i < urls.length; i++) { + String seed = urls[i].trim(); + if (!seed.startsWith("http://") && !seed.startsWith("https://")) { + urls[i] = (isSslEnabled ? "https://" : "http://") + seed; + } + } + } + private Map processCompatibleProperties(Map props) { // Compatible with ES catalog properties Map properties = Maps.newHashMap(props); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java index 5d771638577c67..b9dbc6ff645916 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsTable.java @@ -151,8 +151,6 @@ public List needCompatDateFields() { private void validate(Map properties) throws DdlException { EsResource.valid(properties, false); - hosts = properties.get(EsResource.HOSTS).trim(); - seeds = hosts.split(","); if (properties.containsKey(EsResource.USER)) { userName = properties.get(EsResource.USER).trim(); } @@ -198,6 +196,12 @@ private void validate(Map properties) throws DdlException { maxDocValueFields = DEFAULT_MAX_DOCVALUE_FIELDS; } } + + hosts = properties.get(EsResource.HOSTS).trim(); + seeds = hosts.split(","); + // parse httpSslEnabled before use it here. + EsResource.fillUrlsWithSchema(seeds, httpSslEnabled); + tableContext.put("hosts", hosts); tableContext.put("userName", userName); tableContext.put("passwd", passwd); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java index 650e5374e5fb6d..52f9bc984fa031 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java @@ -70,7 +70,11 @@ private Map processCompatibleProperties(Map prop public String[] getNodes() { String hosts = catalogProperty.getOrDefault(EsResource.HOSTS, ""); - return hosts.trim().split(","); + String sslEnabled = + catalogProperty.getOrDefault(EsResource.HTTP_SSL_ENABLED, EsResource.HTTP_SSL_ENABLED_DEFAULT_VALUE); + String[] hostUrls = hosts.trim().split(","); + EsResource.fillUrlsWithSchema(hostUrls, Boolean.parseBoolean(sslEnabled)); + return hostUrls; } public String getUsername() { diff --git a/regression-test/data/es_p0/test_es_query_no_http_url.out b/regression-test/data/es_p0/test_es_query_no_http_url.out new file mode 100644 index 00000000000000..b195d6d1875b7a --- /dev/null +++ b/regression-test/data/es_p0/test_es_query_no_http_url.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql51 -- +[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] + +-- !sql52 -- +[2020-01-01, 2020-01-02] [-1, 0, 1, 2] [0, 1, 2, 3] ['d', 'e', 'f'] [128, 129, -129, -130] ['192.168.0.1', '127.0.0.1'] string1 [1, 2, 3, 4] 2022-08-08 2022-08-08T12:10:10 text#1 [2020-01-01, 2020-01-02] 3.14 [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] ['a', 'b', 'c'] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] 2022-08-08T12:10:10 2022-08-08T12:10:10 2022-08-08T20:10:10 [1, -2, -3, 4] [1, 0, 1, 1] [32768, 32769, -32769, -32770] + +-- !sql61 -- +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01 00:00:00, 2020-01-02 00:00:00] ['2020-01-01 12:00:00', '2020-01-02 13:01:01'] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 + +-- !sql71 -- +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 + +-- !sql81 -- +[1, 0, 1, 1] [1, -2, -3, 4] [2020-01-01, 2020-01-02] [2020-01-01 12:00:00, 2020-01-02 13:01:01] [1, 2, 3, 4] [1, 1.1, 1.2, 1.3] [1, 2, 3, 4] [32768, 32769, -32769, -32770] ['192.168.0.1', '127.0.0.1'] ['a', 'b', 'c'] [-1, 0, 1, 2] ['{"name":"Andy","age":18}', '{"name":"Tim","age":28}'] [1, 2, 3, 4] [128, 129, -129, -130] ['d', 'e', 'f'] [0, 1, 2, 3] string1 text#1 3.14 2022-08-08T00:00 2022-08-08T12:10:10 1659931810000 2022-08-08T12:10:10 2022-08-08T20:10:10 + diff --git a/regression-test/suites/es_p0/test_es_query_no_http_url.groovy b/regression-test/suites/es_p0/test_es_query_no_http_url.groovy new file mode 100644 index 00000000000000..26516266723a5b --- /dev/null +++ b/regression-test/suites/es_p0/test_es_query_no_http_url.groovy @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_es_query_no_http_url", "p0") { + + String enabled = context.config.otherConfigs.get("enableEsTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String es_6_port = context.config.otherConfigs.get("es_6_port") + String es_7_port = context.config.otherConfigs.get("es_7_port") + String es_8_port = context.config.otherConfigs.get("es_8_port") + + sql """drop catalog if exists es6;""" + sql """drop catalog if exists es7;""" + sql """drop catalog if exists es8;""" + sql """drop resource if exists es6_resource;""" + sql """drop resource if exists es7_resource;""" + sql """drop resource if exists es8_resource;""" + sql """drop table if exists test_v1;""" + sql """drop table if exists test_v2;""" + + // test old create-catalog syntax for compatibility + sql """ + create catalog es6 + properties ( + "type"="es", + "elasticsearch.hosts"="127.0.0.1:$es_6_port", + "elasticsearch.nodes_discovery"="false", + "elasticsearch.keyword_sniff"="true" + ); + """ + + // test new create catalog syntax + sql """create resource if not exists es7_resource properties( + "type"="es", + "hosts"="127.0.0.1:$es_7_port", + "nodes_discovery"="false", + "enable_keyword_sniff"="true" + ); + """ + + sql """create resource if not exists es8_resource properties( + "type"="es", + "hosts"="127.0.0.1:$es_8_port", + "nodes_discovery"="false", + "enable_keyword_sniff"="true" + ); + """ + + // test external table for datetime + sql """ + CREATE TABLE `test_v1` ( + `c_datetime` array NULL, + `c_long` array NULL, + `c_unsigned_long` array NULL, + `c_text` array NULL, + `c_short` array NULL, + `c_ip` array NULL, + `test1` text NULL, + `c_half_float` array NULL, + `test4` date NULL, + `test5` datetime NULL, + `test2` text NULL, + `c_date` array NULL, + `test3` double NULL, + `c_scaled_float` array NULL, + `c_float` array NULL, + `c_double` array NULL, + `c_keyword` array NULL, + `c_person` array NULL, + `test6` datetime NULL, + `test7` datetime NULL, + `test8` datetime NULL, + `c_byte` array NULL, + `c_bool` array NULL, + `c_integer` array NULL + ) ENGINE=ELASTICSEARCH + COMMENT 'ELASTICSEARCH' + PROPERTIES ( + "hosts" = "127.0.0.1:$es_7_port", + "index" = "test1", + "nodes_discovery"="false", + "enable_keyword_sniff"="true", + "http_ssl_enabled"="false" + ); + """ + order_qt_sql51 """select * from test_v1 where test2='text#1'""" + + sql """ + CREATE TABLE `test_v2` ( + `c_datetime` array NULL, + `c_long` array NULL, + `c_unsigned_long` array NULL, + `c_text` array NULL, + `c_short` array NULL, + `c_ip` array NULL, + `test1` text NULL, + `c_half_float` array NULL, + `test4` datev2 NULL, + `test5` datetimev2 NULL, + `test2` text NULL, + `c_date` array NULL, + `test3` double NULL, + `c_scaled_float` array NULL, + `c_float` array NULL, + `c_double` array NULL, + `c_keyword` array NULL, + `c_person` array NULL, + `test6` datetimev2 NULL, + `test7` datetimev2 NULL, + `test8` datetimev2 NULL, + `c_byte` array NULL, + `c_bool` array NULL, + `c_integer` array NULL + ) ENGINE=ELASTICSEARCH + COMMENT 'ELASTICSEARCH' + PROPERTIES ( + "hosts" = "127.0.0.1:$es_8_port", + "index" = "test1", + "nodes_discovery"="false", + "enable_keyword_sniff"="true", + "http_ssl_enabled"="false" + ); + """ + order_qt_sql52 """select * from test_v2 where test2='text#1'""" + + sql """create catalog if not exists es6 with resource es6_resource;""" + sql """create catalog if not exists es7 with resource es7_resource;""" + sql """create catalog if not exists es8 with resource es8_resource;""" + // es6 + sql """switch es6""" + order_qt_sql61 """select * from test1 where test2='text#1'""" + // es7 + sql """switch es7""" + order_qt_sql71 """select * from test1 where test2='text#1'""" + // es8 + sql """switch es8""" + order_qt_sql81 """select * from test1 where test2='text#1'""" + + sql """drop catalog if exists es6;""" + sql """drop catalog if exists es7;""" + sql """drop catalog if exists es8;""" + sql """drop resource if exists es6_resource;""" + sql """drop resource if exists es7_resource;""" + sql """drop resource if exists es8_resource;""" + } +} From 5e06c2b8a4cc4654b3ed8985b8f8a5886cc52021 Mon Sep 17 00:00:00 2001 From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com> Date: Fri, 28 Jul 2023 09:53:55 +0800 Subject: [PATCH 4/9] [Fix](EsCatalog) fix be core when query the table of Es catalog with null fields (#22279) --- be/src/exec/es/es_scroll_parser.cpp | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index 6a0587e48c72cc..5c0c640dd1200f 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -147,7 +147,7 @@ static Status get_int_value(const rapidjson::Value& col, PrimitiveType type, voi return Status::OK(); } - if (pure_doc_value && col.IsArray()) { + if (pure_doc_value && col.IsArray() && !col.Empty()) { RETURN_ERROR_IF_COL_IS_NOT_NUMBER(col[0], type); *reinterpret_cast(slot) = (T)(sizeof(T) < 8 ? col[0].GetInt() : col[0].GetInt64()); return Status::OK(); @@ -246,7 +246,7 @@ static Status get_date_int(const rapidjson::Value& col, PrimitiveType type, bool // processing date type field, if a number is encountered, Doris On ES will force it to be processed according to ms // Doris On ES needs to be consistent with ES, so just divided by 1000 because the unit for from_unixtime is seconds return get_date_value_int(col, type, false, slot); - } else if (col.IsArray() && pure_doc_value) { + } else if (col.IsArray() && pure_doc_value && !col.Empty()) { // this would happened just only when `enable_docvalue_scan = true` // ES add default format for all field after ES 6.4, if we not provided format for `date` field ES would impose // a standard date-format for date field as `2020-06-16T00:00:00.000Z` @@ -282,7 +282,7 @@ static Status get_float_value(const rapidjson::Value& col, PrimitiveType type, v return Status::OK(); } - if (pure_doc_value && col.IsArray()) { + if (pure_doc_value && col.IsArray() && !col.Empty()) { *reinterpret_cast(slot) = (T)(sizeof(T) == 4 ? col[0].GetFloat() : col[0].GetDouble()); return Status::OK(); } @@ -310,7 +310,7 @@ static Status insert_float_value(const rapidjson::Value& col, PrimitiveType type return Status::OK(); } - if (pure_doc_value && col.IsArray() && nullable) { + if (pure_doc_value && col.IsArray() && !col.Empty() && nullable) { T value = (T)(sizeof(T) == 4 ? col[0].GetFloat() : col[0].GetDouble()); col_ptr->insert_data(const_cast(reinterpret_cast(&value)), 0); return Status::OK(); @@ -339,7 +339,7 @@ static Status insert_int_value(const rapidjson::Value& col, PrimitiveType type, return Status::OK(); } - if (pure_doc_value && col.IsArray()) { + if (pure_doc_value && col.IsArray() && !col.Empty()) { RETURN_ERROR_IF_COL_IS_NOT_NUMBER(col[0], type); T value = (T)(sizeof(T) < 8 ? col[0].GetInt() : col[0].GetInt64()); col_ptr->insert_data(const_cast(reinterpret_cast(&value)), 0); @@ -476,7 +476,7 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, // because of reading value from _source, we can not process all json type and then just transfer the value to original string representation // this may be a tricky, but we can workaround this issue std::string val; - if (pure_doc_value) { + if (pure_doc_value && !col.Empty()) { if (!col[0].IsString()) { val = json_value_to_string(col[0]); } else { @@ -546,11 +546,11 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, } bool is_nested_str = false; - if (pure_doc_value && col.IsArray() && col[0].IsBool()) { + if (pure_doc_value && col.IsArray() && !col.Empty() && col[0].IsBool()) { int8_t val = col[0].GetBool(); col_ptr->insert_data(const_cast(reinterpret_cast(&val)), 0); break; - } else if (pure_doc_value && col.IsArray() && col[0].IsString()) { + } else if (pure_doc_value && col.IsArray() && !col.Empty() && col[0].IsString()) { is_nested_str = true; } else if (pure_doc_value && col.IsArray()) { return Status::InternalError(ERROR_INVALID_COL_DATA, "BOOLEAN"); @@ -575,7 +575,7 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, data.assign_from_double(col.GetDouble()); } else { std::string val; - if (pure_doc_value) { + if (pure_doc_value && !col.Empty()) { if (!col[0].IsString()) { val = json_value_to_string(col[0]); } else { @@ -619,7 +619,7 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, case TYPE_VARCHAR: case TYPE_STRING: { std::string val; - if (pure_doc_value) { + if (pure_doc_value && !sub_col.Empty()) { if (!sub_col[0].IsString()) { val = json_value_to_string(sub_col[0]); } else { @@ -695,10 +695,12 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, } bool is_nested_str = false; - if (pure_doc_value && sub_col.IsArray() && sub_col[0].IsBool()) { + if (pure_doc_value && sub_col.IsArray() && !sub_col.Empty() && + sub_col[0].IsBool()) { array.push_back(sub_col[0].GetBool()); break; - } else if (pure_doc_value && sub_col.IsArray() && sub_col[0].IsString()) { + } else if (pure_doc_value && sub_col.IsArray() && !sub_col.Empty() && + sub_col[0].IsString()) { is_nested_str = true; } else if (pure_doc_value && sub_col.IsArray()) { return Status::InternalError(ERROR_INVALID_COL_DATA, "BOOLEAN"); From 968cca20571c65b0355170e8544cdbd437d07b0f Mon Sep 17 00:00:00 2001 From: qiye Date: Tue, 1 Aug 2023 21:45:16 +0800 Subject: [PATCH 5/9] [fix](DOE) es catalog not working with pipeline,datetimev2, array and esquery (#22046) --- be/src/exec/es/es_scroll_parser.cpp | 24 +++++-------- .../external/elasticsearch/QueryBuilders.java | 5 +++ regression-test/data/es_p0/test_es_query.out | 34 +++++++++++++++++++ .../suites/es_p0/test_es_query.groovy | 6 ++++ 4 files changed, 53 insertions(+), 16 deletions(-) diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index 5c0c640dd1200f..fb0d2c76071d46 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -188,7 +188,7 @@ static Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type cctz::parse("%Y-%m-%dT%H:%M:%E*S%Ez", str_date, cctz::utc_time_zone(), &tp); if (ok) { success = dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), - cctz::local_time_zone().name()); + cctz::local_time_zone()); } } else if (str_length == 19) { // YYYY-MM-DDTHH:MM:SS @@ -198,7 +198,7 @@ static Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type cctz::parse("%Y-%m-%dT%H:%M:%S", str_date, cctz::utc_time_zone(), &tp); if (ok) { success = dt_val.from_unixtime(std::chrono::system_clock::to_time_t(tp), - cctz::local_time_zone().name()); + cctz::local_time_zone()); } } else { // YYYY-MM-DD HH:MM:SS @@ -209,7 +209,7 @@ static Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type // string long like "1677895728000" int64_t time_long = std::atol(str_date.c_str()); if (time_long > 0) { - success = dt_val.from_unixtime(time_long / 1000, cctz::local_time_zone().name()); + success = dt_val.from_unixtime(time_long / 1000, cctz::local_time_zone()); } } else { // YYYY-MM-DD or others @@ -221,7 +221,7 @@ static Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type } } else { - if (!dt_val.from_unixtime(col.GetInt64() / 1000, cctz::local_time_zone().name())) { + if (!dt_val.from_unixtime(col.GetInt64() / 1000, cctz::local_time_zone())) { RETURN_ERROR_IF_CAST_FORMAT_ERROR(col, type); } } @@ -619,19 +619,11 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, case TYPE_VARCHAR: case TYPE_STRING: { std::string val; - if (pure_doc_value && !sub_col.Empty()) { - if (!sub_col[0].IsString()) { - val = json_value_to_string(sub_col[0]); - } else { - val = sub_col[0].GetString(); - } + RETURN_ERROR_IF_COL_IS_ARRAY(sub_col, sub_type); + if (!sub_col.IsString()) { + val = json_value_to_string(sub_col); } else { - RETURN_ERROR_IF_COL_IS_ARRAY(sub_col, type); - if (!sub_col.IsString()) { - val = json_value_to_string(sub_col); - } else { - val = sub_col.GetString(); - } + val = sub_col.GetString(); } array.push_back(val); break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java index a0a805747a53e0..8dc8d4abea9574 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java @@ -224,6 +224,11 @@ public static QueryBuilder toEsDsl(Expr expr, List notPushDownList, Map= '2022-08-08 00:00:00' and test4 < '2022-08-08 23:59:59'""" order_qt_sql83 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test1""" order_qt_sql84 """select c_bool[1], c_byte[1], c_short[1], c_integer[1], c_long[1], c_unsigned_long[1], c_float[1], c_half_float[1], c_double[1], c_scaled_float[1], c_date[1], c_datetime[1], c_keyword[1], c_text[1], c_ip[1], c_person[1] from test2""" order_qt_sql85 """select * from test1 where esquery(test2, '{"match":{"test2":"text#1"}}')""" + order_qt_sql86 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test1""" + order_qt_sql87 """select c_bool, c_byte, c_short, c_integer, c_long, c_unsigned_long, c_float, c_half_float, c_double, c_scaled_float, c_date, c_datetime, c_keyword, c_text, c_ip, c_person from test2""" sql """drop catalog if exists es6;""" From eaf149ec17f435f9893c0589f3047b6c3f2de23e Mon Sep 17 00:00:00 2001 From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com> Date: Fri, 4 Aug 2023 11:28:30 +0800 Subject: [PATCH 6/9] [BugFix](Es Catalog) fix bug that es catalog will return error when query partial columns (#22423) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug： When the value of some ES column is empty, querying these value in doc_values mode will receive an error. Reson： In doc values mode, these values are empty, We need to determine if the array is empty --- be/src/exec/es/es_scroll_parser.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index fb0d2c76071d46..720bd50215ab71 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -476,8 +476,10 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, // because of reading value from _source, we can not process all json type and then just transfer the value to original string representation // this may be a tricky, but we can workaround this issue std::string val; - if (pure_doc_value && !col.Empty()) { - if (!col[0].IsString()) { + if (pure_doc_value) { + if (col.Empty()) { + break; + } else if (!col[0].IsString()) { val = json_value_to_string(col[0]); } else { val = col[0].GetString(); @@ -575,8 +577,10 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc, data.assign_from_double(col.GetDouble()); } else { std::string val; - if (pure_doc_value && !col.Empty()) { - if (!col[0].IsString()) { + if (pure_doc_value) { + if (col.Empty()) { + break; + } else if (!col[0].IsString()) { val = json_value_to_string(col[0]); } else { val = col[0].GetString(); From cccc12bf5b2544031dac22d051cf2f2fff3f6384 Mon Sep 17 00:00:00 2001 From: qiye Date: Mon, 7 Aug 2023 18:43:01 +0800 Subject: [PATCH 7/9] (fix)[DOE]fix buildtin functions defination --- gensrc/script/doris_builtins_functions.py | 100 ++++++++++++++++------ 1 file changed, 75 insertions(+), 25 deletions(-) diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 33f5dfb771dba0..25454849a613b9 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -2155,31 +2155,81 @@ [['coalesce'], 'STRING', ['STRING', '...'], '', '', '', 'vec', 'CUSTOM'], # esquery's first arg is suitable for all types - [['esquery'], 'BOOLEAN', ['BOOLEAN', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['TINYINT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['SMALLINT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['INT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['BIGINT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['LARGEINT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['FLOAT', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['DOUBLE', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['DATE', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['DATETIME', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['CHAR', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['VARCHAR', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['JSONB', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['DECIMALV2', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['DECIMAL32', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['DECIMAL64', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['DECIMAL128', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['TIME', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['DATEV2', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['DATETIMEV2', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['TIMEV2', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['ARRAY', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['MAP', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['STRING', 'VARCHAR'], ''], - [['esquery'], 'BOOLEAN', ['VARIANT', 'VARCHAR'], ''], + [['esquery'], 'BOOLEAN', ['BOOLEAN', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['TINYINT', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['SMALLINT', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['INT', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['BIGINT', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['LARGEINT', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['FLOAT', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['DOUBLE', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['DATE', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['DATETIME', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['CHAR', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['VARCHAR', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['JSONB', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['DECIMALV2', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['DECIMAL32', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['DECIMAL64', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['DECIMAL128', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['TIME', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['DATEV2', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['DATETIMEV2', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['TIMEV2', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['ARRAY', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['MAP', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['STRING', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], + [['esquery'], 'BOOLEAN', ['VARIANT', 'VARCHAR'], + '_ZN5doris11ESFunctions5matchEPN' + '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], # String builtin functions [['substr', 'substring'], 'VARCHAR', ['VARCHAR', 'INT'], From 805f5336df6b4f3418e9de614c4c30df8c1d239e Mon Sep 17 00:00:00 2001 From: qiye Date: Mon, 7 Aug 2023 18:53:41 +0800 Subject: [PATCH 8/9] fix code format --- be/src/runtime/thread_context.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/runtime/thread_context.cpp b/be/src/runtime/thread_context.cpp index 371067279164bd..6b422aca6fdf2a 100644 --- a/be/src/runtime/thread_context.cpp +++ b/be/src/runtime/thread_context.cpp @@ -59,7 +59,7 @@ AttachTask::~AttachTask() { SwitchThreadMemTrackerLimiter::SwitchThreadMemTrackerLimiter( const std::shared_ptr& mem_tracker) { - SwitchBthreadLocal::switch_to_bthread_local(); + SwitchBthreadLocal::switch_to_bthread_local(); _old_mem_tracker = thread_context()->thread_mem_tracker_mgr->limiter_mem_tracker(); thread_context()->thread_mem_tracker_mgr->attach_limiter_tracker(mem_tracker, TUniqueId()); } From 164e09a14f39f5c41ec84edda48c650fa91aebab Mon Sep 17 00:00:00 2001 From: qiye Date: Mon, 7 Aug 2023 20:39:34 +0800 Subject: [PATCH 9/9] remove unsupported variant --- gensrc/script/doris_builtins_functions.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 25454849a613b9..2a9ab95ecd101f 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -2227,9 +2227,6 @@ [['esquery'], 'BOOLEAN', ['STRING', 'VARCHAR'], '_ZN5doris11ESFunctions5matchEPN' '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], - [['esquery'], 'BOOLEAN', ['VARIANT', 'VARCHAR'], - '_ZN5doris11ESFunctions5matchEPN' - '9doris_udf15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', ''], # String builtin functions [['substr', 'substring'], 'VARCHAR', ['VARCHAR', 'INT'],