From 4420dbbb03162b4422fccb57263feb134c57bf57 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 9 Apr 2025 15:42:11 -0700 Subject: [PATCH 01/10] [fix](hive) support hive serde dialect for FE --- .../apache/doris/analysis/ArrayLiteral.java | 2 +- .../apache/doris/analysis/BoolLiteral.java | 3 +- .../org/apache/doris/analysis/MapLiteral.java | 2 +- .../apache/doris/analysis/StructLiteral.java | 2 +- .../apache/doris/common/FormatOptions.java | 27 +++++++++++++-- .../apache/doris/nereids/NereidsPlanner.java | 4 ++- .../serde/test_serde_dialect_hive.out | 8 ++++- .../serde/test_serde_dialect_hive.groovy | 33 ++++++++++++++++++- 8 files changed, 71 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java index 2e0f43758f8e79..b6aa82d8ccdeea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java @@ -152,7 +152,7 @@ public String getStringValueInFe(FormatOptions options) { // we should use type to decide we output array is suitable for json format list.add(stringLiteral); }); - return "[" + StringUtils.join(list, ", ") + "]"; + return "[" + StringUtils.join(list, options.getCollectionDelim()) + "]"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java index c0d6d885285151..521dc1bd00bc69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java @@ -113,7 +113,8 @@ public String getStringValue() { @Override public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + String val = options.isBoolValueNum() ? getStringValue() : (value ? "true" : "false"); + return options.getNestedStringWrapper() + val + options.getNestedStringWrapper(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java index f330dfa6edb8bc..b7bc7ce61167fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java @@ -193,7 +193,7 @@ public String getStringValueInFe(FormatOptions options) { list.add(getStringLiteralForComplexType(children.get(i), options) + options.getMapKeyDelim() + getStringLiteralForComplexType(children.get(i + 1), options)); } - return "{" + StringUtils.join(list, ", ") + "}"; + return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java index 5d888168821d2f..afc6a495af6441 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java @@ -125,7 +125,7 @@ public String getStringValueInFe(FormatOptions options) { + options.getMapKeyDelim() + getStringLiteralForComplexType(child, options)); } - return "{" + StringUtils.join(list, ", ") + "}"; + return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java b/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java index a63b83ab71da98..e89bbc9f011374 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java @@ -28,11 +28,20 @@ public class FormatOptions { private String nestedStringWrapper; private String mapKeyDelim; private String nullFormat; + private String collectionDelim; + // isBoolValue = true means the boolean column in collection type(array, map, ...) will print as 0 or 1. + // false means to print as true/false + // This is only for boolean column within the collection type. + // For top level boolean column, it is always 0/1 + private boolean isBoolValueNum; - public FormatOptions(String nestedStringWrapper, String mapKeyDelim, String nullFormat) { + private FormatOptions(String nestedStringWrapper, String mapKeyDelim, String nullFormat, String collectionDelim, + boolean isBoolValueNum) { this.nestedStringWrapper = nestedStringWrapper; this.mapKeyDelim = mapKeyDelim; this.nullFormat = nullFormat; + this.collectionDelim = collectionDelim; + this.isBoolValueNum = isBoolValueNum; } public String getNestedStringWrapper() { @@ -47,11 +56,23 @@ public String getNullFormat() { return this.nullFormat; } + public String getCollectionDelim() { + return collectionDelim; + } + + public boolean isBoolValueNum() { + return isBoolValueNum; + } + public static FormatOptions getDefault() { - return new FormatOptions("\"", ":", "null"); + return new FormatOptions("\"", ":", "null", ", ", true); } public static FormatOptions getForPresto() { - return new FormatOptions("", "=", "NULL"); + return new FormatOptions("", "=", "NULL", ", ", true); + } + + public static FormatOptions getForHive() { + return new FormatOptions("\"", "=", "null", ",", false); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 5f6b74a597b8bb..0a96e577d74f1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -809,8 +809,10 @@ private void setFormatOptions() { case "trino": statementContext.setFormatOptions(FormatOptions.getForPresto()); break; - case "doris": case "hive": + statementContext.setFormatOptions(FormatOptions.getForHive()); + break; + case "doris": statementContext.setFormatOptions(FormatOptions.getDefault()); break; default: diff --git a/regression-test/data/datatype_p0/serde/test_serde_dialect_hive.out b/regression-test/data/datatype_p0/serde/test_serde_dialect_hive.out index 3ea1043cdf6f9c..aebc2e62a3b815 100644 --- a/regression-test/data/datatype_p0/serde/test_serde_dialect_hive.out +++ b/regression-test/data/datatype_p0/serde/test_serde_dialect_hive.out @@ -2,6 +2,12 @@ -- !sql01 -- 1 2 3 4 5 1.1 2.0 123456.123456789 2024-06-30 2024-06-30T10:10:11 2024-06-30T10:10:11.123456 59.50.185.152 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff this is a string with , and " abc ef 123ndedwdw true [1,2,3,4,5] [1, 2, 3, null, 5] [1.1, 2.1, 3.1, null, 5] [1.10000, 2.10000, 3.00000, null, 5.12345] ["abc", "de, f"", null, ""] [{"k1":"v1", "k2":null, "k3":"", "k4":"a , "a"}, {"k1":"v1", "k2":null, "k3 , "abc":"", "k4":"a , "a"}] [["abc", "de, f"", null, ""], [], null] \N \N {"k1":"v1", "k2":null, "k3":"", "k4":"a , "a"} {"k1":[["abc", "de, f"", null, ""], [], null], "k2":null} {10:{"k1":[["abc", "de, f"", null, ""], [], null]}, 11:null} \N {"s_id":100, "s_name":"abc , "", "s_address":null} {"s_id":null, "s_name":["abc", "de, f"", null, ""], "s_address":""} ["2024-06-01", null, "2024-06-03"] ["2024-06-01 10:10:10.000", null, "2024-06-03 01:11:23.123"] [1, 1, 0, 0, 1, 0, 0] {"s_id":100, "s_name":"abc , "", "s_gender":1} {"k1":0, "k2":1, "k3":1, "k4":0} --- !sql01 -- +-- !sql_fe01 -- +1 2 3 4 5 1.1 2.0000 123456.123456789 2024-06-30 2024-06-30 10:10:11 2024-06-30 10:10:11.123456 59.50.185.152 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff this is a string with , and " abc ef 123ndedwdw true [1, 2, 3, 4, 5] [1, 2, 3, null, 5] [1.10, 2.10, 3.10, null, 5.00] [1.10000, 2.10000, 3.00000, null, 5.12345] ["abc", "de, f"", null, ""] [{"k1":"v1", "k2":null, "k3":"", "k4":"a , "a"}, {"k1":"v1", "k2":null, "k3 , "abc":"", "k4":"a , "a"}] [["abc", "de, f"", null, ""], [], null] {"k1":"v1", "k2":null, "k3":"", "k4":"a , "a"} {"k1":[["abc", "de, f"", null, ""], [], null], "k2":null} {10:{"k1":[["abc", "de, f"", null, ""], [], null]}, 11:null} {"s_id":100, "s_name":"abc , "", "s_address":null} {"s_id":null, "s_name":["abc", "de, f"", null, ""], "s_address":""} ["2024-06-01", null, "2024-06-03"] ["2024-06-01 10:10:10", null, "2024-06-03 01:11:23.123"] [1, 1, 0, 0, 1, 0, 0] {"s_id":100, "s_name":"abc , "", "s_gender":1} {"k1":0, "k2":1, "k3":1, "k4":0} + +-- !sql02 -- 1 2 3 4 5 1.1 2.0 123456.123456789 2024-06-30 2024-06-30T10:10:11 2024-06-30T10:10:11.123456 59.50.185.152 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff this is a string with , and " abc ef 123ndedwdw true [1,2,3,4,5] [1,2,3,null,5] [1.1,2.1,3.1,null,5] [1.10000,2.10000,3.00000,null,5.12345] ["abc","de, f"",null,""] [{"k1":"v1","k2":null,"k3":"","k4":"a , "a"},{"k1":"v1","k2":null,"k3 , "abc":"","k4":"a , "a"}] [["abc","de, f"",null,""],[],null] \N \N {"k1":"v1","k2":null,"k3":"","k4":"a , "a"} {"k1":[["abc","de, f"",null,""],[],null],"k2":null} {10:{"k1":[["abc","de, f"",null,""],[],null]},11:null} \N {"s_id":100,"s_name":"abc , "","s_address":null} {"s_id":null,"s_name":["abc","de, f"",null,""],"s_address":""} ["2024-06-01",null,"2024-06-03"] ["2024-06-01 10:10:10.000",null,"2024-06-03 01:11:23.123"] [true,true,false,false,true,false,false] {"s_id":100,"s_name":"abc , "","s_gender":true} {"k1":false,"k2":true,"k3":true,"k4":false} +-- !sql_fe02 -- +1 2 3 4 5 1.1 2.0000 123456.123456789 2024-06-30 2024-06-30 10:10:11 2024-06-30 10:10:11.123456 59.50.185.152 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff this is a string with , and " abc ef 123ndedwdw true [1, 2, 3, 4, 5] [1,2,3,null,5] [1.10,2.10,3.10,null,5.00] [1.10000,2.10000,3.00000,null,5.12345] ["abc","de, f"",null,""] [{"k1":"v1","k2":null,"k3":"","k4":"a , "a"},{"k1":"v1","k2":null,"k3 , "abc":"","k4":"a , "a"}] [["abc","de, f"",null,""],[],null] {"k1":"v1","k2":null,"k3":"","k4":"a , "a"} {"k1":[["abc","de, f"",null,""],[],null],"k2":null} {10:{"k1":[["abc","de, f"",null,""],[],null]},11:null} {"s_id":100,"s_name":"abc , "","s_address":null} {"s_id":null,"s_name":["abc","de, f"",null,""],"s_address":""} ["2024-06-01",null,"2024-06-03"] ["2024-06-01 10:10:10",null,"2024-06-03 01:11:23.123"] [true,true,false,false,true,false,false] {"s_id":100,"s_name":"abc , "","s_gender":true} {"k1":false,"k2":true,"k3":true,"k4":false} + diff --git a/regression-test/suites/datatype_p0/serde/test_serde_dialect_hive.groovy b/regression-test/suites/datatype_p0/serde/test_serde_dialect_hive.groovy index b8e3037d770f7d..5e9724a8b0ac1a 100644 --- a/regression-test/suites/datatype_p0/serde/test_serde_dialect_hive.groovy +++ b/regression-test/suites/datatype_p0/serde/test_serde_dialect_hive.groovy @@ -95,10 +95,41 @@ suite("test_serde_dialect_hive", "p0") { ); """ + String constant_sql=""" + select 1,2,3,4,5,1.1,2.0000,123456.123456789,"2024-06-30", "2024-06-30 10:10:11", "2024-06-30 10:10:11.123456", + '59.50.185.152', + 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', + 'this is a string with , and "', + 'abc ef', + ' 123ndedwdw', + true, + '[1, 2, 3, 4, 5]', + [1,2,3,null,5], + [1.1,2.1,3.1,null,5.00], + [1.1,2.1,3.00000,null,5.12345], + ['abc', 'de, f"', null, ''], + [{'k1': 'v1', 'k2': null, 'k3':'', 'k4':'a , "a'}, {'k1': 'v1', 'k2': null, 'k3 , "abc':'', 'k4':'a , "a'}], + [['abc', 'de, f"', null, ''],[],null], + {'k1': 'v1', 'k2': null, 'k3':'', 'k4':'a , "a'}, + {'k1': [['abc', 'de, f"', null, ''],[],null], 'k2': null}, + {10: {'k1': [['abc', 'de, f"', null, ''],[],null]}, 11: null}, + named_struct('s_id', 100, 's_name', 'abc , "', 's_address', null), + named_struct('s_id', null, 's_name', ['abc', 'de, f"', null, ''], 's_address', ''), + ['2024-06-01',null,'2024-06-03'], + ['2024-06-01 10:10:10',null,'2024-06-03 01:11:23.123'], + [true, true, false, false, true, false, false], + named_struct('s_id', 100, 's_name', 'abc , "', 's_gender', true), + {'k1': false, 'k2': true, 'k3':true, 'k4': false} + """ + sql """set serde_dialect="doris";""" qt_sql01 """select * from test_serde_dialect_hive_tbl""" + // test fold in FE + qt_sql_fe01 """${constant_sql}""" sql """set serde_dialect="hive";""" - qt_sql01 """select * from test_serde_dialect_hive_tbl""" + qt_sql02 """select * from test_serde_dialect_hive_tbl""" + // test fold in FE + qt_sql_fe02 """${constant_sql}""" test { sql """set serde_dialect="invalid"""" From 7209539bede82547dc176ae943b2bec09307be96 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 10 Apr 2025 11:53:16 +0800 Subject: [PATCH 02/10] 2 --- .../java/org/apache/doris/analysis/BoolLiteral.java | 2 +- .../java/org/apache/doris/analysis/LiteralExpr.java | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java index 521dc1bd00bc69..2c58c3965ba798 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java @@ -114,7 +114,7 @@ public String getStringValue() { @Override public String getStringValueForArray(FormatOptions options) { String val = options.isBoolValueNum() ? getStringValue() : (value ? "true" : "false"); - return options.getNestedStringWrapper() + val + options.getNestedStringWrapper(); + return val; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 301b6277725247..643b8ab2f53f0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -114,10 +114,16 @@ public static LiteralExpr create(String value, Type type) throws AnalysisExcepti return literalExpr; } + /** + * 1. For numeric/complex type, no need to wrap with quota, call "getStringValueInFe". + * 2. For other type, call "getStringValueForArray()": + * 2.1. for null/boolean, getStringValueForArray() will return format value in FormatOptions. + * 2.2. for others, getStringValueForArray() will return value wrapped with quota. + */ public static String getStringLiteralForComplexType(Expr v, FormatOptions options) { if (!(v instanceof NullLiteral) && v.getType().isScalarType() - && (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType())) - || v.getType() == Type.BOOLEAN)) { + && Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))) { + // This is a numeric type, no need to wrap with quota, so call getStringValueInFe return v.getStringValueInFe(options); } else if (v.getType().isComplexType()) { // these type should also call getStringValueInFe which should handle special case for itself From ad694e6855df5745d012af76df51d0a4df440b69 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 9 Apr 2025 21:32:08 -0700 Subject: [PATCH 03/10] 2 --- .../java/org/apache/doris/analysis/LiteralExpr.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 643b8ab2f53f0b..87841a69900929 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -114,12 +114,12 @@ public static LiteralExpr create(String value, Type type) throws AnalysisExcepti return literalExpr; } - /** - * 1. For numeric/complex type, no need to wrap with quota, call "getStringValueInFe". - * 2. For other type, call "getStringValueForArray()": - * 2.1. for null/boolean, getStringValueForArray() will return format value in FormatOptions. - * 2.2. for others, getStringValueForArray() will return value wrapped with quota. - */ + /** + * 1. For numeric/complex type, no need to wrap with quota, call "getStringValueInFe". + * 2. For other type, call "getStringValueForArray()": + * 2.1. for null/boolean, getStringValueForArray() will return format value in FormatOptions. + * 2.2. for others, getStringValueForArray() will return value wrapped with quota. + */ public static String getStringLiteralForComplexType(Expr v, FormatOptions options) { if (!(v instanceof NullLiteral) && v.getType().isScalarType() && Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))) { From 37054317c093c309bdc126d682fe60cd06939409 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 10 Apr 2025 14:51:53 -0700 Subject: [PATCH 04/10] 4 --- .../apache/doris/analysis/ArrayLiteral.java | 9 +- .../apache/doris/analysis/BoolLiteral.java | 3 +- .../org/apache/doris/analysis/CastExpr.java | 4 +- .../apache/doris/analysis/DateLiteral.java | 4 +- .../apache/doris/analysis/DecimalLiteral.java | 5 - .../java/org/apache/doris/analysis/Expr.java | 12 +- .../apache/doris/analysis/FloatLiteral.java | 9 +- .../apache/doris/analysis/IPv4Literal.java | 4 +- .../apache/doris/analysis/IPv6Literal.java | 4 +- .../org/apache/doris/analysis/IntLiteral.java | 6 - .../apache/doris/analysis/JsonLiteral.java | 2 +- .../doris/analysis/LargeIntLiteral.java | 6 - .../apache/doris/analysis/LiteralExpr.java | 25 +-- .../org/apache/doris/analysis/MapLiteral.java | 15 +- .../org/apache/doris/analysis/MaxLiteral.java | 2 +- .../apache/doris/analysis/NullLiteral.java | 2 +- .../doris/analysis/PlaceHolderExpr.java | 4 +- .../apache/doris/analysis/StringLiteral.java | 4 +- .../apache/doris/analysis/StructLiteral.java | 11 +- .../apache/doris/common/FormatOptions.java | 2 +- .../plans/commands/insert/InsertUtils.java | 2 +- .../doris/analysis/ArrayLiteralTest.java | 162 ++++++++---------- .../apache/doris/analysis/MapLiteralTest.java | 155 ++++++----------- .../doris/analysis/StructLiteralTest.java | 55 ++---- 24 files changed, 181 insertions(+), 326 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java index b6aa82d8ccdeea..b9cab6a5d7dcaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java @@ -132,13 +132,6 @@ public String getStringValue() { return "[" + StringUtils.join(list, ", ") + "]"; } - @Override - public String getStringValueForArray(FormatOptions options) { - List list = new ArrayList<>(children.size()); - children.forEach(v -> list.add(v.getStringValueForArray(options))); - return "[" + StringUtils.join(list, ", ") + "]"; - } - @Override public String getStringValueInFe(FormatOptions options) { List list = new ArrayList<>(children.size()); @@ -147,7 +140,7 @@ public String getStringValueInFe(FormatOptions options) { if (v instanceof NullLiteral) { stringLiteral = options.getNullFormat(); } else { - stringLiteral = getStringLiteralForComplexType(v, options); + stringLiteral = v.getStringValueForComplexType(options); } // we should use type to decide we output array is suitable for json format list.add(stringLiteral); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java index 2c58c3965ba798..acc3b9528083ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java @@ -111,8 +111,9 @@ public String getStringValue() { return value ? "1" : "0"; } + @Override - public String getStringValueForArray(FormatOptions options) { + public String getStringValueInFe(FormatOptions options) { String val = options.isBoolValueNum() ? getStringValue() : (value ? "true" : "false"); return val; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index 75bc129b523d0a..8942303a94196c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -577,8 +577,8 @@ public boolean isNullable() { } @Override - public String getStringValueForArray(FormatOptions options) { - return children.get(0).getStringValueForArray(options); + public String getStringValueForComplexType(FormatOptions options) { + return children.get(0).getStringValueForComplexType(options); } public void setNotFold(boolean notFold) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 0ea9a5dc23e912..ce51f05b700550 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -765,8 +765,8 @@ public String getStringValue(Type type) { } @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + public String getStringValueForComplexType(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueInFe(options) + options.getNestedStringWrapper(); } public void roundCeiling(int newScale) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java index 6fbfc175ea039f..fbaba5063e0d74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java @@ -280,11 +280,6 @@ public String getStringValue() { return value.toPlainString(); } - @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); - } - @Override public long getLongValue() { return value.longValue(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index 0b5e68e1f30346..4d08c03d28767c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -2185,6 +2185,13 @@ public String getStringValue() { return ""; } + /** + * This method is used for constant fold in FE, + * for different serde dialect(hive, presto, doris). + * + * @param options + * @return + */ public String getStringValueInFe(FormatOptions options) { return getStringValue(); } @@ -2198,7 +2205,10 @@ public String getStringValueForStreamLoad(FormatOptions options) { // ["1", "2", "3"] // ["a", "b", "c"] // [["1", "2", "3"], ["1"], ["3"]] - public String getStringValueForArray(FormatOptions options) { + // This method is to return the string value of this expr in a complex type. + // For most of the integer types, it is same as getStringValueInFe(). + // But for others like StringLiteral and DateLiteral, it should be wrapped with quotations. + public String getStringValueForComplexType(FormatOptions options) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java index 645afe52f99491..5a92b248ad442f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java @@ -170,13 +170,12 @@ public String getStringValueInFe(FormatOptions options) { } @Override - public String getStringValueForArray(FormatOptions options) { - String ret = getStringValue(); + public String getStringValueForComplexType(FormatOptions options) { + String ret = getStringValueInFe(options); if (type == Type.TIME || type == Type.TIMEV2) { - // here already wrapped in '' - ret = ret.substring(1, ret.length() - 1); + ret = options.getNestedStringWrapper() + ret + options.getNestedStringWrapper(); } - return options.getNestedStringWrapper() + ret + options.getNestedStringWrapper(); + return ret; } public static Type getDefaultTimeType(Type type) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java index 92063e6b9b30c3..e1ef6a076899e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java @@ -140,7 +140,7 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + public String getStringValueForComplexType(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueInFe(options) + options.getNestedStringWrapper(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java index bb986e0ffe7a2c..9e32b4c2facead 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java @@ -105,7 +105,7 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + public String getStringValueForComplexType(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueInFe(options) + options.getNestedStringWrapper(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java index 978c864434dce4..a33a1b741ec8a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java @@ -20,7 +20,6 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.FormatOptions; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.util.ByteBufferUtil; import org.apache.doris.qe.ConnectContext; @@ -284,11 +283,6 @@ public String getStringValue() { return Long.toString(value); } - @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); - } - @Override public long getLongValue() { return value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java index e568fafe45aa5e..5e7ad601ac2119 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java @@ -101,7 +101,7 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { + public String getStringValueForComplexType(FormatOptions options) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java index 861aeaa043efcf..f499d58e046823 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java @@ -20,7 +20,6 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.FormatOptions; import org.apache.doris.common.io.Text; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; @@ -197,11 +196,6 @@ public String getStringValue() { return value.toString(); } - @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); - } - @Override public long getLongValue() { return value.longValue(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 87841a69900929..52fd8bf7202c81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -114,25 +114,6 @@ public static LiteralExpr create(String value, Type type) throws AnalysisExcepti return literalExpr; } - /** - * 1. For numeric/complex type, no need to wrap with quota, call "getStringValueInFe". - * 2. For other type, call "getStringValueForArray()": - * 2.1. for null/boolean, getStringValueForArray() will return format value in FormatOptions. - * 2.2. for others, getStringValueForArray() will return value wrapped with quota. - */ - public static String getStringLiteralForComplexType(Expr v, FormatOptions options) { - if (!(v instanceof NullLiteral) && v.getType().isScalarType() - && Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))) { - // This is a numeric type, no need to wrap with quota, so call getStringValueInFe - return v.getStringValueInFe(options); - } else if (v.getType().isComplexType()) { - // these type should also call getStringValueInFe which should handle special case for itself - return v.getStringValueInFe(options); - } else { - return v.getStringValueForArray(options); - } - } - public static String getStringLiteralForStreamLoad(Expr v, FormatOptions options) { if (!(v instanceof NullLiteral) && v.getType().isScalarType() && (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType())) @@ -142,7 +123,7 @@ public static String getStringLiteralForStreamLoad(Expr v, FormatOptions options // these type should also call getStringValueInFe which should handle special case for itself return v.getStringValueForStreamLoad(options); } else { - return v.getStringValueForArray(options); + return v.getStringValueForComplexType(options); } } @@ -276,7 +257,9 @@ public String getStringValueInFe(FormatOptions options) { } @Override - public abstract String getStringValueForArray(FormatOptions options); + public String getStringValueForComplexType(FormatOptions options) { + return getStringValueInFe(options); + } public long getLongValue() { return 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java index b7bc7ce61167fd..388363a00516a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java @@ -170,17 +170,6 @@ private String getStringValue(Expr expr) { return expr.getStringValue(); } - @Override - public String getStringValueForArray(FormatOptions options) { - List list = new ArrayList<>(children.size()); - for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) { - list.add(children.get(i).getStringValueForArray(options) - + options.getMapKeyDelim() - + children.get(i + 1).getStringValueForArray(options)); - } - return "{" + StringUtils.join(list, ", ") + "}"; - } - @Override public String getStringValueInFe(FormatOptions options) { List list = new ArrayList<>(children.size()); @@ -190,8 +179,8 @@ public String getStringValueInFe(FormatOptions options) { // map key type do not support complex type throw new UnsupportedOperationException("Unsupported key type for MAP: " + children.get(i).getType()); } - list.add(getStringLiteralForComplexType(children.get(i), options) - + options.getMapKeyDelim() + getStringLiteralForComplexType(children.get(i + 1), options)); + list.add(children.get(i).getStringValueForComplexType(options) + + options.getMapKeyDelim() + children.get(i + 1).getStringValueForComplexType(options)); } return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java index f3f2f24a77cd96..6fa0d42f151395 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java @@ -69,7 +69,7 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { + public String getStringValueForComplexType(FormatOptions options) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java index ff784b4546a609..02cfccf4a743a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java @@ -113,7 +113,7 @@ public String getStringValueInFe(FormatOptions options) { // the null value inside an array is represented as "null", for exampe: // [null, null]. Not same as other primitive type to represent as \N. @Override - public String getStringValueForArray(FormatOptions options) { + public String getStringValueForComplexType(FormatOptions options) { return options.getNullFormat(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java index 0e5aeeb362b5fd..53d26913a73990 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java @@ -191,8 +191,8 @@ public Expr reset() { } @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + public String getStringValueForComplexType(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueInFe(options) + options.getNestedStringWrapper(); } public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index 09cb50bb9ed72b..64871911f75b58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -154,8 +154,8 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + public String getStringValueForComplexType(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueInFe(options) + options.getNestedStringWrapper(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java index afc6a495af6441..2e775d87c7fa38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java @@ -106,13 +106,6 @@ public String getStringValue() { return "{" + StringUtils.join(list, ", ") + "}"; } - @Override - public String getStringValueForArray(FormatOptions options) { - List list = new ArrayList<>(children.size()); - children.forEach(v -> list.add(v.getStringValueForArray(options))); - return "{" + StringUtils.join(list, ", ") + "}"; - } - @Override public String getStringValueInFe(FormatOptions options) { List list = new ArrayList<>(children.size()); @@ -123,7 +116,7 @@ public String getStringValueInFe(FormatOptions options) { + ((StructType) type).getFields().get(i).getName() + options.getNestedStringWrapper() + options.getMapKeyDelim() - + getStringLiteralForComplexType(child, options)); + + child.getStringValueForComplexType(options)); } return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } @@ -131,7 +124,7 @@ public String getStringValueInFe(FormatOptions options) { @Override public String getStringValueForStreamLoad(FormatOptions options) { List list = new ArrayList<>(children.size()); - children.forEach(v -> list.add(getStringLiteralForComplexType(v, options))); + children.forEach(v -> v.getStringValueForComplexType(options)); return "{" + StringUtils.join(list, ", ") + "}"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java b/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java index e89bbc9f011374..71bf9a9dfd4703 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java @@ -73,6 +73,6 @@ public static FormatOptions getForPresto() { } public static FormatOptions getForHive() { - return new FormatOptions("\"", "=", "null", ",", false); + return new FormatOptions("\"", ":", "null", ",", false); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index e4b38a311c0844..46263036a509e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -180,7 +180,7 @@ public static InternalService.PDataRow getRowStringValue(List c row.addColBuilder().setValue(StmtExecutor.NULL_VALUE_FOR_LOAD); } else if (expr instanceof ArrayLiteral) { row.addColBuilder().setValue(String.format("\"%s\"", - ((ArrayLiteral) expr).toLegacyLiteral().getStringValueForArray(options))); + ((ArrayLiteral) expr).toLegacyLiteral().getStringValueForComplexType(options))); } else { row.addColBuilder().setValue(String.format("\"%s\"", ((Literal) expr).toLegacyLiteral().getStringValue())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java index d1d7a3c736e779..148bfbf80fa353 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java @@ -26,10 +26,13 @@ public class ArrayLiteralTest { @Test - public void testGetStringValueForArray() throws AnalysisException { + public void testGetStringInFe() throws AnalysisException { FormatOptions options = FormatOptions.getDefault(); IntLiteral intLiteral1 = new IntLiteral(1); FloatLiteral floatLiteral = new FloatLiteral("2.15"); + FloatLiteral floatLiteral1 = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), + FloatLiteral.getDefaultTimeType(Type.TIME)); + BoolLiteral boolLiteral = new BoolLiteral(true); StringLiteral stringLiteral = new StringLiteral("shortstring"); LargeIntLiteral largeIntLiteral = new LargeIntLiteral("1000000000000000000000"); @@ -37,97 +40,66 @@ public void testGetStringValueForArray() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[\"1\", \"2.15\"]", arrayLiteral1.getStringValueForArray(options)); + Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueInFe(options)); + ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); + Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueInFe(options)); ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); - Assert.assertEquals("[\"1\", \"1\"]", arrayLiteral2.getStringValueForArray(options)); + Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueInFe(options)); ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueForArray(options)); + Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueInFe(options)); ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); - Assert.assertEquals("[\"1000000000000000000000\", \"1000000000000000000000\"]", arrayLiteral4.getStringValueForArray(options)); + Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", + arrayLiteral4.getStringValueInFe(options)); ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueForArray(options)); + Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueInFe(options)); ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueForArray(options)); + Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueInFe(options)); ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); - Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]", arrayLiteral7.getStringValueForArray(options)); + Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]", + arrayLiteral7.getStringValueInFe(options)); ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); Assert.assertEquals("[[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"], [\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]]", - arrayLiteral8.getStringValueForArray(options)); + arrayLiteral8.getStringValueInFe(options)); ArrayLiteral arrayLiteral9 = new ArrayLiteral(); - Assert.assertEquals("[]", arrayLiteral9.getStringValueForArray(options)); - - ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); - MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); - StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); - ArrayLiteral arrayLiteral10 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[\"1\", \"2.15\"], [\"1\", \"2.15\"]]", arrayLiteral10.getStringValueForArray(options)); - ArrayLiteral arrayLiteral11 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{\"1\":\"2.15\"}]", arrayLiteral11.getStringValueForArray(options)); - ArrayLiteral arrayLiteral12 = new ArrayLiteral(structLiteral); - Assert.assertEquals("[{\"1\", \"2.15\", \"2022-10-10\"}]", arrayLiteral12.getStringValueForArray(options)); - } - - @Test - public void testGetStringValueForArrayForPresto() throws AnalysisException { - FormatOptions options = FormatOptions.getForPresto(); - IntLiteral intLiteral1 = new IntLiteral(1); - FloatLiteral floatLiteral = new FloatLiteral("2.15"); - BoolLiteral boolLiteral = new BoolLiteral(true); - StringLiteral stringLiteral = new StringLiteral("shortstring"); - LargeIntLiteral largeIntLiteral = new LargeIntLiteral("1000000000000000000000"); - NullLiteral nullLiteral = new NullLiteral(); - DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); - DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); - ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1, 2.15]", arrayLiteral1.getStringValueForArray(options)); - - ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); - Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueForArray(options)); - - ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[shortstring, shortstring]", arrayLiteral3.getStringValueForArray(options)); - - ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); - Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", arrayLiteral4.getStringValueForArray(options)); - - ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[NULL, NULL]", arrayLiteral5.getStringValueForArray(options)); - - ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[2022-10-10, 2022-10-10]", arrayLiteral6.getStringValueForArray(options)); - - ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); - Assert.assertEquals("[2022-10-10 12:10:10, 2022-10-10 12:10:10]", arrayLiteral7.getStringValueForArray(options)); + Assert.assertEquals("[]", arrayLiteral9.getStringValueInFe(options)); - ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); - Assert.assertEquals("[[2022-10-10 12:10:10, 2022-10-10 12:10:10], [2022-10-10 12:10:10, 2022-10-10 12:10:10]]", - arrayLiteral8.getStringValueForArray(options)); + DecimalLiteral decimalLiteral = new DecimalLiteral("1.0"); + DecimalLiteral decimalLiteral2 = new DecimalLiteral("2"); + ArrayLiteral arrayLiteral10 = new ArrayLiteral(decimalLiteral, decimalLiteral2); + Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueInFe(options)); - ArrayLiteral arrayLiteral9 = new ArrayLiteral(); - Assert.assertEquals("[]", arrayLiteral9.getStringValueForArray(options)); + //array(1, null) + IntLiteral intLiteralWithNull = new IntLiteral(1); + ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); + Assert.assertEquals("[1, null]", arrayLiteral11.getStringValueInFe(options)); + //array(null, 1) + ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); + Assert.assertEquals("[null, 1]", arrayLiteral12.getStringValueInFe(options)); ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); - ArrayLiteral arrayLiteral10 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[1, 2.15], [1, 2.15]]", arrayLiteral10.getStringValueForArray(options)); - ArrayLiteral arrayLiteral11 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{1=2.15}]", arrayLiteral11.getStringValueForArray(options)); - ArrayLiteral arrayLiteral12 = new ArrayLiteral(structLiteral); - Assert.assertEquals("[{1, 2.15, 2022-10-10}]", arrayLiteral12.getStringValueForArray(options)); + ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); + Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", + arrayLiteral13.getStringValueInFe(options)); + ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); + Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueInFe(options)); + ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); + Assert.assertEquals("[{\"col1\":1, \"col2\":2.15, \"col3\":\"2022-10-10\"}]", + arrayLiteral15.getStringValueInFe(options)); } @Test - public void testGetStringInFe() throws AnalysisException { - FormatOptions options = FormatOptions.getDefault(); + public void testGetStringInFeForPresto() throws AnalysisException { + FormatOptions options = FormatOptions.getForPresto(); IntLiteral intLiteral1 = new IntLiteral(1); FloatLiteral floatLiteral = new FloatLiteral("2.15"); FloatLiteral floatLiteral1 = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), @@ -142,30 +114,30 @@ public void testGetStringInFe() throws AnalysisException { ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueInFe(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); - Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueInFe(options)); + Assert.assertEquals("[11:22:33]", arrayLiteralWithTime.getStringValueInFe(options)); ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueInFe(options)); ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueInFe(options)); + Assert.assertEquals("[shortstring, shortstring]", arrayLiteral3.getStringValueInFe(options)); ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", arrayLiteral4.getStringValueInFe(options)); ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueInFe(options)); + Assert.assertEquals("[NULL, NULL]", arrayLiteral5.getStringValueInFe(options)); ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueInFe(options)); + Assert.assertEquals("[2022-10-10, 2022-10-10]", arrayLiteral6.getStringValueInFe(options)); ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); - Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]", + Assert.assertEquals("[2022-10-10 12:10:10, 2022-10-10 12:10:10]", arrayLiteral7.getStringValueInFe(options)); ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); - Assert.assertEquals("[[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"], [\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]]", + Assert.assertEquals("[[2022-10-10 12:10:10, 2022-10-10 12:10:10], [2022-10-10 12:10:10, 2022-10-10 12:10:10]]", arrayLiteral8.getStringValueInFe(options)); ArrayLiteral arrayLiteral9 = new ArrayLiteral(); @@ -179,25 +151,25 @@ public void testGetStringInFe() throws AnalysisException { //array(1, null) IntLiteral intLiteralWithNull = new IntLiteral(1); ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); - Assert.assertEquals("[1, null]", arrayLiteral11.getStringValueInFe(options)); + Assert.assertEquals("[1, NULL]", arrayLiteral11.getStringValueInFe(options)); //array(null, 1) ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); - Assert.assertEquals("[null, 1]", arrayLiteral12.getStringValueInFe(options)); + Assert.assertEquals("[NULL, 1]", arrayLiteral12.getStringValueInFe(options)); ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[\"1\", \"2.15\"], [\"1\", \"2.15\"]]", arrayLiteral13.getStringValueForArray(options)); + Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", arrayLiteral13.getStringValueInFe(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{\"1\":\"2.15\"}]", arrayLiteral14.getStringValueForArray(options)); + Assert.assertEquals("[{1=2.15}]", arrayLiteral14.getStringValueInFe(options)); ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); - Assert.assertEquals("[{\"1\", \"2.15\", \"2022-10-10\"}]", arrayLiteral15.getStringValueForArray(options)); + Assert.assertEquals("[{col1=1, col2=2.15, col3=2022-10-10}]", arrayLiteral15.getStringValueInFe(options)); } @Test - public void testGetStringInFeForPresto() throws AnalysisException { - FormatOptions options = FormatOptions.getForPresto(); + public void testGetStringInFeForHive() throws AnalysisException { + FormatOptions options = FormatOptions.getForHive(); IntLiteral intLiteral1 = new IntLiteral(1); FloatLiteral floatLiteral = new FloatLiteral("2.15"); FloatLiteral floatLiteral1 = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), @@ -210,32 +182,33 @@ public void testGetStringInFeForPresto() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueInFe(options)); + Assert.assertEquals("[1.0,2.15]", arrayLiteral1.getStringValueInFe(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); - Assert.assertEquals("[11:22:33]", arrayLiteralWithTime.getStringValueInFe(options)); + Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueInFe(options)); ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); - Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueInFe(options)); + Assert.assertEquals("[true,true]", arrayLiteral2.getStringValueInFe(options)); ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[shortstring, shortstring]", arrayLiteral3.getStringValueInFe(options)); + Assert.assertEquals("[\"shortstring\",\"shortstring\"]", arrayLiteral3.getStringValueInFe(options)); ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); - Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", + Assert.assertEquals("[1000000000000000000000,1000000000000000000000]", arrayLiteral4.getStringValueInFe(options)); ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[NULL, NULL]", arrayLiteral5.getStringValueInFe(options)); + Assert.assertEquals("[null,null]", arrayLiteral5.getStringValueInFe(options)); ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[2022-10-10, 2022-10-10]", arrayLiteral6.getStringValueInFe(options)); + Assert.assertEquals("[\"2022-10-10\",\"2022-10-10\"]", arrayLiteral6.getStringValueInFe(options)); ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); - Assert.assertEquals("[2022-10-10 12:10:10, 2022-10-10 12:10:10]", + Assert.assertEquals("[\"2022-10-10 12:10:10\",\"2022-10-10 12:10:10\"]", arrayLiteral7.getStringValueInFe(options)); ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); - Assert.assertEquals("[[2022-10-10 12:10:10, 2022-10-10 12:10:10], [2022-10-10 12:10:10, 2022-10-10 12:10:10]]", + Assert.assertEquals( + "[[\"2022-10-10 12:10:10\",\"2022-10-10 12:10:10\"],[\"2022-10-10 12:10:10\",\"2022-10-10 12:10:10\"]]", arrayLiteral8.getStringValueInFe(options)); ArrayLiteral arrayLiteral9 = new ArrayLiteral(); @@ -244,24 +217,25 @@ public void testGetStringInFeForPresto() throws AnalysisException { DecimalLiteral decimalLiteral = new DecimalLiteral("1.0"); DecimalLiteral decimalLiteral2 = new DecimalLiteral("2"); ArrayLiteral arrayLiteral10 = new ArrayLiteral(decimalLiteral, decimalLiteral2); - Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueInFe(options)); + Assert.assertEquals("[1.0,2.0]", arrayLiteral10.getStringValueInFe(options)); //array(1, null) IntLiteral intLiteralWithNull = new IntLiteral(1); ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); - Assert.assertEquals("[1, NULL]", arrayLiteral11.getStringValueInFe(options)); + Assert.assertEquals("[1,null]", arrayLiteral11.getStringValueInFe(options)); //array(null, 1) ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); - Assert.assertEquals("[NULL, 1]", arrayLiteral12.getStringValueInFe(options)); + Assert.assertEquals("[null,1]", arrayLiteral12.getStringValueInFe(options)); ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[1, 2.15], [1, 2.15]]", arrayLiteral13.getStringValueForArray(options)); + Assert.assertEquals("[[1.0,2.15],[1.0,2.15]]", arrayLiteral13.getStringValueInFe(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{1=2.15}]", arrayLiteral14.getStringValueForArray(options)); + Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueInFe(options)); ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); - Assert.assertEquals("[{1, 2.15, 2022-10-10}]", arrayLiteral15.getStringValueForArray(options)); + Assert.assertEquals("[{\"col1\":1,\"col2\":2.15,\"col3\":\"2022-10-10\"}]", + arrayLiteral15.getStringValueInFe(options)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java index c36d9fcc0d2583..3c15fe3ad5f9ff 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java @@ -58,34 +58,35 @@ public static void setUp() throws AnalysisException { arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); mapLiteral = new MapLiteral(intLiteral1, floatLiteral); structLiteral = new StructLiteral(intLiteral1, floatLiteral, decimalLiteral1, dateLiteral); - } @Test - public void testGetStringValueForArray() throws AnalysisException { + public void testGetStringInFe() throws AnalysisException { FormatOptions options = FormatOptions.getDefault(); MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{\"1\":\"2.15\"}", mapLiteral1.getStringValueForArray(options)); + Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueInFe(options)); + MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); + Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueInFe(options)); MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{\"1\":\"shortstring\"}", mapLiteral2.getStringValueForArray(options)); + Assertions.assertEquals("{1:\"shortstring\"}", mapLiteral2.getStringValueInFe(options)); MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{\"1000000000000000000000\":\"2022-10-10\"}", mapLiteral3.getStringValueForArray(options)); - MapLiteral mapLiteral4 = new MapLiteral(nullLiteral, nullLiteral); - Assertions.assertEquals("{null:null}", mapLiteral4.getStringValueForArray(options)); + Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", mapLiteral3.getStringValueInFe(options)); + MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); + Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueInFe(options)); MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); - Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", - mapLiteral5.getStringValueForArray(options)); - - MapLiteral mapLiteral6 = new MapLiteral(); - Assertions.assertEquals("{}", mapLiteral6.getStringValueForArray(options)); + Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", mapLiteral5.getStringValueInFe(options)); + MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); + Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueInFe(options)); - MapLiteral mapLiteral7 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{null:\"1\"}", mapLiteral7.getStringValueForArray(options)); - MapLiteral mapLiteral8 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{\"1\":null}", mapLiteral8.getStringValueForArray(options)); + MapLiteral mapLiteral7 = new MapLiteral(); + Assertions.assertEquals("{}", mapLiteral7.getStringValueInFe(options)); + MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); + Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueInFe(options)); + MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); + Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueInFe(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{\"1\":[\"1\", \"2.15\"]}", mapLiteral10.getStringValueForArray(options)); + Assertions.assertEquals("{1:[1.0, 2.15]}", mapLiteral10.getStringValueInFe(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -93,8 +94,8 @@ public void testGetStringValueForArray() throws AnalysisException { + "detailMessage = Invalid key type in Map, not support array", e.getMessage()); } - MapLiteral mapLiteral11 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{\"1.0\":{\"1\":\"2.15\"}}", mapLiteral11.getStringValueForArray(options)); + MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); + Assertions.assertEquals("{1.0:{1:2.15}}", mapLiteral12.getStringValueInFe(options)); try { new MapLiteral(mapLiteral, decimalLiteral1); } catch (Exception e) { @@ -103,94 +104,44 @@ public void testGetStringValueForArray() throws AnalysisException { } MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); - Assertions.assertEquals("{\"shortstring\":{\"1\", \"2.15\", \"1.0\", \"2022-10-10\"}}", - mapLiteral13.getStringValueForArray(options)); + Assertions.assertEquals("{\"shortstring\":{\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", + mapLiteral13.getStringValueInFe(options)); try { new MapLiteral(structLiteral, stringLiteral); - } catch (Exception e) { - Assertions.assertEquals("errCode = 2, detailMessage = Invalid key type in Map, " - + "not support struct", e.getMessage()); - } - } - - @Test - public void testGetStringValueForArrayForPresto() throws AnalysisException { - FormatOptions options = FormatOptions.getForPresto(); - MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{1=2.15}", mapLiteral1.getStringValueForArray(options)); - MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{1=shortstring}", mapLiteral2.getStringValueForArray(options)); - MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{1000000000000000000000=2022-10-10}", mapLiteral3.getStringValueForArray(options)); - MapLiteral mapLiteral4 = new MapLiteral(nullLiteral, nullLiteral); - Assertions.assertEquals("{NULL=NULL}", mapLiteral4.getStringValueForArray(options)); - MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); - Assertions.assertEquals("{2022-10-10 12:10:10=2022-10-10}", mapLiteral5.getStringValueForArray(options)); - - MapLiteral mapLiteral6 = new MapLiteral(); - Assertions.assertEquals("{}", mapLiteral6.getStringValueForArray(options)); - - MapLiteral mapLiteral7 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{NULL=1}", mapLiteral7.getStringValueForArray(options)); - MapLiteral mapLiteral8 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{1=NULL}", mapLiteral8.getStringValueForArray(options)); - - MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1=[1, 2.15]}", mapLiteral10.getStringValueForArray(options)); - try { - new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { Assertions.assertEquals("errCode = 2, " - + "detailMessage = Invalid key type in Map, not support array", e.getMessage()); - } - - MapLiteral mapLiteral11 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{1.0={1=2.15}}", mapLiteral11.getStringValueForArray(options)); - try { - new MapLiteral(mapLiteral, decimalLiteral1); - } catch (Exception e) { - Assertions.assertEquals("errCode = 2, " - + "detailMessage = Invalid key type in Map, not support map", e.getMessage()); - } - - MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); - Assertions.assertEquals("{shortstring={1, 2.15, 1.0, 2022-10-10}}", - mapLiteral13.getStringValueForArray(options)); - try { - new MapLiteral(structLiteral, stringLiteral); - } catch (Exception e) { - Assertions.assertEquals("errCode = 2, detailMessage = Invalid key type in Map, " + + "detailMessage = Invalid key type in Map, " + "not support struct", e.getMessage()); } } @Test - public void testGetStringInFe() throws AnalysisException { - FormatOptions options = FormatOptions.getDefault(); + public void testGetStringInFeForPresto() throws AnalysisException { + FormatOptions options = FormatOptions.getForPresto(); MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueInFe(options)); + Assertions.assertEquals("{1=2.15}", mapLiteral1.getStringValueInFe(options)); MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); - Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueInFe(options)); + Assertions.assertEquals("{1=11:22:33}", mapLiteral11.getStringValueInFe(options)); MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{1:\"shortstring\"}", mapLiteral2.getStringValueInFe(options)); + Assertions.assertEquals("{1=shortstring}", mapLiteral2.getStringValueInFe(options)); MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", mapLiteral3.getStringValueInFe(options)); + Assertions.assertEquals("{1000000000000000000000=2022-10-10}", mapLiteral3.getStringValueInFe(options)); MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); - Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueInFe(options)); + Assertions.assertEquals("{11:22:33=NULL}", mapLiteral4.getStringValueInFe(options)); MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); - Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", mapLiteral5.getStringValueInFe(options)); + Assertions.assertEquals("{2022-10-10 12:10:10=2022-10-10}", mapLiteral5.getStringValueInFe(options)); MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); - Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueInFe(options)); + Assertions.assertEquals("{1.0=2}", mapLiteral6.getStringValueInFe(options)); MapLiteral mapLiteral7 = new MapLiteral(); Assertions.assertEquals("{}", mapLiteral7.getStringValueInFe(options)); MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueInFe(options)); + Assertions.assertEquals("{NULL=1}", mapLiteral8.getStringValueInFe(options)); MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueInFe(options)); + Assertions.assertEquals("{1=NULL}", mapLiteral9.getStringValueInFe(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{\"1\":[\"1\", \"2.15\"]}", mapLiteral10.getStringValueForArray(options)); + Assertions.assertEquals("{1=[1.0, 2.15]}", mapLiteral10.getStringValueInFe(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -199,7 +150,7 @@ public void testGetStringInFe() throws AnalysisException { } MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{\"1.0\":{\"1\":\"2.15\"}}", mapLiteral12.getStringValueForArray(options)); + Assertions.assertEquals("{1.0={1=2.15}}", mapLiteral12.getStringValueInFe(options)); try { new MapLiteral(mapLiteral, decimalLiteral1); } catch (Exception e) { @@ -208,8 +159,8 @@ public void testGetStringInFe() throws AnalysisException { } MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); - Assertions.assertEquals("{\"shortstring\":{\"1\", \"2.15\", \"1.0\", \"2022-10-10\"}}", - mapLiteral13.getStringValueForArray(options)); + Assertions.assertEquals("{shortstring={col1=1, col2=2.15, col3=1.0, col4=2022-10-10}}", + mapLiteral13.getStringValueInFe(options)); try { new MapLiteral(structLiteral, stringLiteral); } catch (Exception e) { @@ -220,32 +171,32 @@ public void testGetStringInFe() throws AnalysisException { } @Test - public void testGetStringInFeForPresto() throws AnalysisException { - FormatOptions options = FormatOptions.getForPresto(); + public void testGetStringInFeForHive() throws AnalysisException { + FormatOptions options = FormatOptions.getForHive(); MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{1=2.15}", mapLiteral1.getStringValueInFe(options)); + Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueInFe(options)); MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); - Assertions.assertEquals("{1=11:22:33}", mapLiteral11.getStringValueInFe(options)); + Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueInFe(options)); MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{1=shortstring}", mapLiteral2.getStringValueInFe(options)); + Assertions.assertEquals("{true:\"shortstring\"}", mapLiteral2.getStringValueInFe(options)); MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{1000000000000000000000=2022-10-10}", mapLiteral3.getStringValueInFe(options)); + Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", mapLiteral3.getStringValueInFe(options)); MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); - Assertions.assertEquals("{11:22:33=NULL}", mapLiteral4.getStringValueInFe(options)); + Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueInFe(options)); MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); - Assertions.assertEquals("{2022-10-10 12:10:10=2022-10-10}", mapLiteral5.getStringValueInFe(options)); + Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", mapLiteral5.getStringValueInFe(options)); MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); - Assertions.assertEquals("{1.0=2}", mapLiteral6.getStringValueInFe(options)); + Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueInFe(options)); MapLiteral mapLiteral7 = new MapLiteral(); Assertions.assertEquals("{}", mapLiteral7.getStringValueInFe(options)); MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{NULL=1}", mapLiteral8.getStringValueInFe(options)); + Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueInFe(options)); MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{1=NULL}", mapLiteral9.getStringValueInFe(options)); + Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueInFe(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1=[1, 2.15]}", mapLiteral10.getStringValueForArray(options)); + Assertions.assertEquals("{1:[1.0,2.15]}", mapLiteral10.getStringValueInFe(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -254,7 +205,7 @@ public void testGetStringInFeForPresto() throws AnalysisException { } MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{1.0={1=2.15}}", mapLiteral12.getStringValueForArray(options)); + Assertions.assertEquals("{1.0:{1:2.15}}", mapLiteral12.getStringValueInFe(options)); try { new MapLiteral(mapLiteral, decimalLiteral1); } catch (Exception e) { @@ -263,8 +214,8 @@ public void testGetStringInFeForPresto() throws AnalysisException { } MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); - Assertions.assertEquals("{shortstring={1, 2.15, 1.0, 2022-10-10}}", - mapLiteral13.getStringValueForArray(options)); + Assertions.assertEquals("{\"shortstring\":{\"col1\":1,\"col2\":2.15,\"col3\":1.0,\"col4\":\"2022-10-10\"}}", + mapLiteral13.getStringValueInFe(options)); try { new MapLiteral(structLiteral, stringLiteral); } catch (Exception e) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java index 7e1702d2f7f6b0..07ef7363f80ccf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java @@ -60,26 +60,6 @@ public static void setUp() throws AnalysisException { structLiteral = new StructLiteral(intLiteral1, floatLiteral, decimalLiteral1, dateLiteral); } - @Test - public void testGetStringValueForArray() throws AnalysisException { - FormatOptions options = FormatOptions.getDefault(); - StructLiteral structLiteral1 = new StructLiteral(intLiteral1, floatLiteral, floatLiteral1, boolLiteral, - stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, - datetimeLiteral); - Assert.assertEquals("{\"1\", \"2.15\", \"11:22:33\", \"1\", \"shortstring\", " - + "\"1000000000000000000000\", \"1.0\", \"2\", \"2022-10-10\", \"2022-10-10 12:10:10\"}", - structLiteral1.getStringValueForArray(options)); - StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); - Assert.assertEquals("{[\"1\", \"2.15\"], {\"1\":\"2.15\"}, {\"1\", \"2.15\", \"1.0\", \"2022-10-10\"}}", - structLiteral2.getStringValueForArray(options)); - StructLiteral structLiteral3 = new StructLiteral(); - Assert.assertEquals("{}", structLiteral3.getStringValueForArray(options)); - - StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); - Assert.assertEquals("{null, \"1\"}", nullStruct.getStringValueForArray(options)); - - } - @Test public void testGetStringInFe() throws AnalysisException { FormatOptions options = FormatOptions.getDefault(); @@ -100,41 +80,40 @@ public void testGetStringInFe() throws AnalysisException { } @Test - public void testGetStringValueForArrayForPreto() throws AnalysisException { + public void testGetStringInFeForPresto() throws AnalysisException { FormatOptions options = FormatOptions.getForPresto(); StructLiteral structLiteral1 = new StructLiteral(intLiteral1, floatLiteral, floatLiteral1, boolLiteral, - stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, - datetimeLiteral); - Assert.assertEquals("{1, 2.15, 11:22:33, 1, shortstring, " - + "1000000000000000000000, 1.0, 2, 2022-10-10, 2022-10-10 12:10:10}", - structLiteral1.getStringValueForArray(options)); + stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, datetimeLiteral); + Assert.assertEquals("{col1=1, col2=2.15, col3=11:22:33, col4=1, col5=" + + "shortstring, col6=1000000000000000000000, col7=1.0, col8=2, col9=2022-10-10, col10=2022-10-10 12:10:10}", + structLiteral1.getStringValueInFe(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); - Assert.assertEquals("{[1, 2.15], {1=2.15}, {1, 2.15, 1.0, 2022-10-10}}", - structLiteral2.getStringValueForArray(options)); + Assert.assertEquals("{col1=[1.0, 2.15], col2={1=2.15}, col3=" + + "{col1=1, col2=2.15, col3=1.0, col4=2022-10-10}}", + structLiteral2.getStringValueInFe(options)); StructLiteral structLiteral3 = new StructLiteral(); - Assert.assertEquals("{}", structLiteral3.getStringValueForArray(options)); + Assert.assertEquals("{}", structLiteral3.getStringValueInFe(options)); StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); - Assert.assertEquals("{NULL, 1}", nullStruct.getStringValueForArray(options)); - + Assert.assertEquals("{col1=NULL, col2=1}", nullStruct.getStringValueInFe(options)); } @Test - public void testGetStringInFeForPresto() throws AnalysisException { - FormatOptions options = FormatOptions.getForPresto(); + public void testGetStringInFeForHive() throws AnalysisException { + FormatOptions options = FormatOptions.getForHive(); StructLiteral structLiteral1 = new StructLiteral(intLiteral1, floatLiteral, floatLiteral1, boolLiteral, stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, datetimeLiteral); - Assert.assertEquals("{col1=1, col2=2.15, col3=11:22:33, col4=1, col5=" - + "shortstring, col6=1000000000000000000000, col7=1.0, col8=2, col9=2022-10-10, col10=2022-10-10 12:10:10}", + Assert.assertEquals( + "{\"col1\":1,\"col2\":2.15,\"col3\":\"11:22:33\",\"col4\":true,\"col5\":\"shortstring\",\"col6\":1000000000000000000000,\"col7\":1.0,\"col8\":2,\"col9\":\"2022-10-10\",\"col10\":\"2022-10-10 12:10:10\"}", structLiteral1.getStringValueInFe(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); - Assert.assertEquals("{col1=[1.0, 2.15], col2={1=2.15}, col3=" - + "{col1=1, col2=2.15, col3=1.0, col4=2022-10-10}}", + Assert.assertEquals( + "{\"col1\":[1.0,2.15],\"col2\":{1:2.15},\"col3\":{\"col1\":1,\"col2\":2.15,\"col3\":1.0,\"col4\":\"2022-10-10\"}}", structLiteral2.getStringValueInFe(options)); StructLiteral structLiteral3 = new StructLiteral(); Assert.assertEquals("{}", structLiteral3.getStringValueInFe(options)); StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); - Assert.assertEquals("{col1=NULL, col2=1}", nullStruct.getStringValueInFe(options)); + Assert.assertEquals("{\"col1\":null,\"col2\":1}", nullStruct.getStringValueInFe(options)); } } From 68b15cf624d2566f74f296d68b61a3416336e79d Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 10 Apr 2025 16:05:04 -0700 Subject: [PATCH 05/10] 3 --- .../apache/doris/analysis/ArrayLiteral.java | 22 +-- .../apache/doris/analysis/BoolLiteral.java | 2 +- .../org/apache/doris/analysis/CastExpr.java | 9 +- .../apache/doris/analysis/DateLiteral.java | 4 +- .../apache/doris/analysis/DecimalLiteral.java | 2 +- .../java/org/apache/doris/analysis/Expr.java | 44 +++-- .../apache/doris/analysis/FloatLiteral.java | 15 +- .../apache/doris/analysis/IPv4Literal.java | 4 +- .../apache/doris/analysis/IPv6Literal.java | 4 +- .../apache/doris/analysis/JsonLiteral.java | 2 +- .../apache/doris/analysis/LiteralExpr.java | 19 +- .../org/apache/doris/analysis/MapLiteral.java | 21 ++- .../org/apache/doris/analysis/MaxLiteral.java | 2 +- .../apache/doris/analysis/NullLiteral.java | 10 +- .../doris/analysis/PlaceHolderExpr.java | 4 +- .../apache/doris/analysis/StringLiteral.java | 4 +- .../apache/doris/analysis/StructLiteral.java | 12 +- .../apache/doris/common/FormatOptions.java | 1 + .../plans/commands/insert/InsertUtils.java | 11 +- .../physical/PhysicalOneRowRelation.java | 3 +- .../apache/doris/planner/OriginalPlanner.java | 2 +- .../org/apache/doris/qe/StmtExecutor.java | 17 +- .../doris/analysis/ArrayLiteralTest.java | 175 +++++++++++++----- .../doris/analysis/DateLiteralTest.java | 4 +- .../doris/analysis/DecimalLiteralTest.java | 5 +- .../doris/analysis/FloatLiteralTest.java | 8 +- .../apache/doris/analysis/MapLiteralTest.java | 143 +++++++++----- .../doris/analysis/StructLiteralTest.java | 43 +++-- 28 files changed, 373 insertions(+), 219 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java index b9cab6a5d7dcaf..06f5e9fe2e8fea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java @@ -133,17 +133,10 @@ public String getStringValue() { } @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { List list = new ArrayList<>(children.size()); children.forEach(v -> { - String stringLiteral; - if (v instanceof NullLiteral) { - stringLiteral = options.getNullFormat(); - } else { - stringLiteral = v.getStringValueForComplexType(options); - } - // we should use type to decide we output array is suitable for json format - list.add(stringLiteral); + list.add(v.getStringValueInComplexTypeForQuery(options)); }); return "[" + StringUtils.join(list, options.getCollectionDelim()) + "]"; } @@ -152,16 +145,9 @@ public String getStringValueInFe(FormatOptions options) { public String getStringValueForStreamLoad(FormatOptions options) { List list = new ArrayList<>(children.size()); children.forEach(v -> { - String stringLiteral; - if (v instanceof NullLiteral) { - stringLiteral = "null"; - } else { - stringLiteral = getStringLiteralForStreamLoad(v, options); - } - // we should use type to decide we output array is suitable for json format - list.add(stringLiteral); + list.add(v.getStringValueInComplexTypeForStreamLoad(options)); }); - return "[" + StringUtils.join(list, ", ") + "]"; + return "[" + StringUtils.join(list, options.getCollectionDelim()) + "]"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java index acc3b9528083ea..39d91253692f96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java @@ -113,7 +113,7 @@ public String getStringValue() { @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { String val = options.isBoolValueNum() ? getStringValue() : (value ? "true" : "false"); return val; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index 8942303a94196c..437c0e5cae45ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -577,8 +577,13 @@ public boolean isNullable() { } @Override - public String getStringValueForComplexType(FormatOptions options) { - return children.get(0).getStringValueForComplexType(options); + public String getStringValueForStreamLoad(FormatOptions options) { + return children.get(0).getStringValueForStreamLoad(options); + } + + @Override + public String getStringValueInComplexTypeForQuery(FormatOptions options) { + return children.get(0).getStringValueInComplexTypeForQuery(options); } public void setNotFold(boolean notFold) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index ce51f05b700550..69d5ee4a5f201f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -765,8 +765,8 @@ public String getStringValue(Type type) { } @Override - public String getStringValueForComplexType(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValueInFe(options) + options.getNestedStringWrapper(); + public String getStringValueInComplexTypeForQuery(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } public void roundCeiling(int newScale) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java index fbaba5063e0d74..41d64c27ae6ea6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java @@ -266,7 +266,7 @@ public int compareLiteral(LiteralExpr expr) { } @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { return value.toPlainString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index 4d08c03d28767c..0ca9da5904fb22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -2186,30 +2186,42 @@ public String getStringValue() { } /** - * This method is used for constant fold in FE, + * This method is used for constant fold of query in FE, * for different serde dialect(hive, presto, doris). - * - * @param options - * @return */ - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { return getStringValue(); } + /** + * This method is to return the string value of this expr in a complex type for query + * It is only used for "getStringValueForQuery()" + * For most of the integer types, it is same as getStringValueForQuery(). + * But for others like StringLiteral and DateLiteral, it should be wrapped with quotations. + * eg: 1,2,abc,[1,2,3],["abc","def"],{10:20},{"abc":20} + */ + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { + return getStringValueForQuery(options); + } + + /** + * This method is to return the string value of this expr for stream load. + * so there is a little different from "getStringValueForQuery()". + * eg, for NullLiteral, it should be "\N" for stream load, but "null" for FE constant + * for StructLiteral, the value should not contain sub column's name. + */ public String getStringValueForStreamLoad(FormatOptions options) { - return getStringValue(); + return getStringValueForQuery(options); } - // A special method only for array literal, all primitive type in array - // will be wrapped by double quote. eg: - // ["1", "2", "3"] - // ["a", "b", "c"] - // [["1", "2", "3"], ["1"], ["3"]] - // This method is to return the string value of this expr in a complex type. - // For most of the integer types, it is same as getStringValueInFe(). - // But for others like StringLiteral and DateLiteral, it should be wrapped with quotations. - public String getStringValueForComplexType(FormatOptions options) { - return null; + /** + * This method is to return the string value of this expr in a complex type for stream load + * It is only used for "getStringValueForStreamLoad()" + * There is a little different from "getStringValueInComplexTypeForQuery()", + * eg, for StructLiteral, the value should not contain sub column's name. + */ + protected String getStringValueInComplexTypeForStreamLoad(FormatOptions options) { + return getStringValueInComplexTypeForQuery(options); } public final TExpr normalize(Normalizer normalizer) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java index 5a92b248ad442f..732f7eb14055fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java @@ -155,7 +155,7 @@ public String getStringValue() { } @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { if (type == Type.TIME || type == Type.TIMEV2) { // FloatLiteral used to represent TIME type, here we need to remove apostrophe from timeStr // for example '11:22:33' -> 11:22:33 @@ -170,8 +170,17 @@ public String getStringValueInFe(FormatOptions options) { } @Override - public String getStringValueForComplexType(FormatOptions options) { - String ret = getStringValueInFe(options); + public String getStringValueInComplexTypeForQuery(FormatOptions options) { + String ret = this.getStringValueForQuery(options); + if (type == Type.TIME || type == Type.TIMEV2) { + ret = options.getNestedStringWrapper() + ret + options.getNestedStringWrapper(); + } + return ret; + } + + @Override + public String getStringValueForStreamLoad(FormatOptions options) { + String ret = this.getStringValueForQuery(options); if (type == Type.TIME || type == Type.TIMEV2) { ret = options.getNestedStringWrapper() + ret + options.getNestedStringWrapper(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java index e1ef6a076899e7..399e9979b6c1a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java @@ -140,7 +140,7 @@ public String getStringValue() { } @Override - public String getStringValueForComplexType(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValueInFe(options) + options.getNestedStringWrapper(); + public String getStringValueInComplexTypeForQuery(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java index 9e32b4c2facead..2db22de8eac36c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java @@ -105,7 +105,7 @@ public String getStringValue() { } @Override - public String getStringValueForComplexType(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValueInFe(options) + options.getNestedStringWrapper(); + public String getStringValueInComplexTypeForQuery(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java index 5e7ad601ac2119..0a56eff84a0ec0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java @@ -101,7 +101,7 @@ public String getStringValue() { } @Override - public String getStringValueForComplexType(FormatOptions options) { + public String getStringValueInComplexTypeForQuery(FormatOptions options) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 52fd8bf7202c81..76ae813574d8b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -114,19 +114,6 @@ public static LiteralExpr create(String value, Type type) throws AnalysisExcepti return literalExpr; } - public static String getStringLiteralForStreamLoad(Expr v, FormatOptions options) { - if (!(v instanceof NullLiteral) && v.getType().isScalarType() - && (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType())) - || v.getType() == Type.BOOLEAN)) { - return v.getStringValueInFe(options); - } else if (v.getType().isComplexType()) { - // these type should also call getStringValueInFe which should handle special case for itself - return v.getStringValueForStreamLoad(options); - } else { - return v.getStringValueForComplexType(options); - } - } - /** * Init LiteralExpr's Type information * only use in rewrite alias function @@ -252,13 +239,13 @@ public int compareTo(LiteralExpr literalExpr) { @Override public abstract String getStringValue(); - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { return getStringValue(); } @Override - public String getStringValueForComplexType(FormatOptions options) { - return getStringValueInFe(options); + public String getStringValueInComplexTypeForQuery(FormatOptions options) { + return getStringValueForQuery(options); } public long getLongValue() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java index 388363a00516a8..0031777b428722 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java @@ -171,7 +171,7 @@ private String getStringValue(Expr expr) { } @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { List list = new ArrayList<>(children.size()); for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) { // we should use type to decide we output array is suitable for json format @@ -179,8 +179,23 @@ public String getStringValueInFe(FormatOptions options) { // map key type do not support complex type throw new UnsupportedOperationException("Unsupported key type for MAP: " + children.get(i).getType()); } - list.add(children.get(i).getStringValueForComplexType(options) - + options.getMapKeyDelim() + children.get(i + 1).getStringValueForComplexType(options)); + list.add(children.get(i).getStringValueInComplexTypeForQuery(options) + + options.getMapKeyDelim() + children.get(i + 1).getStringValueInComplexTypeForQuery(options)); + } + return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; + } + + @Override + public String getStringValueForStreamLoad(FormatOptions options) { + List list = new ArrayList<>(children.size()); + for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) { + // we should use type to decide we output array is suitable for json format + if (children.get(i).getType().isComplexType()) { + // map key type do not support complex type + throw new UnsupportedOperationException("Unsupported key type for MAP: " + children.get(i).getType()); + } + list.add(children.get(i).getStringValueInComplexTypeForStreamLoad(options) + + options.getMapKeyDelim() + children.get(i + 1).getStringValueInComplexTypeForStreamLoad(options)); } return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java index 6fa0d42f151395..ce6dc0100ec63f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java @@ -69,7 +69,7 @@ public String getStringValue() { } @Override - public String getStringValueForComplexType(FormatOptions options) { + public String getStringValueInComplexTypeForQuery(FormatOptions options) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java index 02cfccf4a743a1..24ec69523c1a95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.FormatOptions; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; @@ -106,14 +107,19 @@ public String getStringValue() { } @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { return null; } + @Override + public String getStringValueForStreamLoad(FormatOptions options) { + return FeConstants.null_string; + } + // the null value inside an array is represented as "null", for exampe: // [null, null]. Not same as other primitive type to represent as \N. @Override - public String getStringValueForComplexType(FormatOptions options) { + public String getStringValueInComplexTypeForQuery(FormatOptions options) { return options.getNullFormat(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java index 53d26913a73990..646f2c9c7bb2e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java @@ -191,8 +191,8 @@ public Expr reset() { } @Override - public String getStringValueForComplexType(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValueInFe(options) + options.getNestedStringWrapper(); + public String getStringValueInComplexTypeForQuery(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index 64871911f75b58..c475ac1b680df1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -154,8 +154,8 @@ public String getStringValue() { } @Override - public String getStringValueForComplexType(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValueInFe(options) + options.getNestedStringWrapper(); + public String getStringValueInComplexTypeForQuery(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java index 2e775d87c7fa38..4c0f16b18e84ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java @@ -107,7 +107,7 @@ public String getStringValue() { } @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { List list = new ArrayList<>(children.size()); // same with be default field index start with 1 for (int i = 0; i < children.size(); i++) { @@ -116,7 +116,7 @@ public String getStringValueInFe(FormatOptions options) { + ((StructType) type).getFields().get(i).getName() + options.getNestedStringWrapper() + options.getMapKeyDelim() - + child.getStringValueForComplexType(options)); + + child.getStringValueInComplexTypeForQuery(options)); } return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } @@ -124,8 +124,12 @@ public String getStringValueInFe(FormatOptions options) { @Override public String getStringValueForStreamLoad(FormatOptions options) { List list = new ArrayList<>(children.size()); - children.forEach(v -> v.getStringValueForComplexType(options)); - return "{" + StringUtils.join(list, ", ") + "}"; + // same with be default field index start with 1 + for (int i = 0; i < children.size(); i++) { + Expr child = children.get(i); + list.add(child.getStringValueInComplexTypeForStreamLoad(options)); + } + return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java b/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java index 71bf9a9dfd4703..fc7e0f4726b719 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java @@ -27,6 +27,7 @@ public class FormatOptions { private String nestedStringWrapper; private String mapKeyDelim; + // the string format of null value in complex type private String nullFormat; private String collectionDelim; // isBoolValue = true means the boolean column in collection type(array, map, ...) will print as 0 or 1. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 46263036a509e3..442c846b67efc1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -53,7 +53,6 @@ import org.apache.doris.nereids.trees.expressions.DefaultValueSlot; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.Plan; @@ -176,15 +175,7 @@ public static InternalService.PDataRow getRowStringValue(List c throw new AnalysisException( "do not support non-literal expr in transactional insert operation: " + expr.toSql()); } - if (expr instanceof NullLiteral) { - row.addColBuilder().setValue(StmtExecutor.NULL_VALUE_FOR_LOAD); - } else if (expr instanceof ArrayLiteral) { - row.addColBuilder().setValue(String.format("\"%s\"", - ((ArrayLiteral) expr).toLegacyLiteral().getStringValueForComplexType(options))); - } else { - row.addColBuilder().setValue(String.format("\"%s\"", - ((Literal) expr).toLegacyLiteral().getStringValue())); - } + row.addColBuilder().setValue(((Literal) expr).toLegacyLiteral().getStringValueForStreamLoad(options)); } return row.build(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java index edde1c17c7a8ad..a053b509b13aa6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java @@ -155,7 +155,8 @@ public Optional computeResultInFe( if (expr instanceof Literal) { LiteralExpr legacyExpr = ((Literal) expr).toLegacyLiteral(); columns.add(new Column(output.getName(), output.getDataType().toCatalogDataType())); - data.add(legacyExpr.getStringValueInFe(cascadesContext.getStatementContext().getFormatOptions())); + data.add(legacyExpr.getStringValueForQuery( + cascadesContext.getStatementContext().getFormatOptions())); } else { return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index feb8c45787a013..e79a934f0468d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -616,7 +616,7 @@ public Optional handleQueryInFe(StatementBase parsedStmt) { String columnName = columnLabels.get(i); if (expr instanceof LiteralExpr) { columns.add(new Column(columnName, expr.getType())); - data.add(((LiteralExpr) expr).getStringValueInFe(options)); + data.add(((LiteralExpr) expr).getStringValueForQuery(options)); } else { return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index f1868176c2c488..4cd9fc50d13215 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -24,7 +24,6 @@ import org.apache.doris.analysis.AnalyzeStmt; import org.apache.doris.analysis.AnalyzeTblStmt; import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.ArrayLiteral; import org.apache.doris.analysis.CreateRoutineLoadStmt; import org.apache.doris.analysis.CreateTableAsSelectStmt; import org.apache.doris.analysis.CreateTableLikeStmt; @@ -43,7 +42,6 @@ import org.apache.doris.analysis.LoadType; import org.apache.doris.analysis.LockTablesStmt; import org.apache.doris.analysis.NativeInsertStmt; -import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.OutFileClause; import org.apache.doris.analysis.PartitionNames; import org.apache.doris.analysis.PlaceHolderExpr; @@ -270,7 +268,6 @@ public class StmtExecutor { private static final AtomicLong STMT_ID_GENERATOR = new AtomicLong(0); public static final int MAX_DATA_TO_SEND_FOR_TXN = 100; - public static final String NULL_VALUE_FOR_LOAD = "\\N"; private static Set blockSqlAstNames = Sets.newHashSet(); private Pattern beIpPattern = Pattern.compile("\\[(\\d+):"); @@ -371,19 +368,7 @@ public static InternalService.PDataRow getRowStringValue(List cols, throw new UserException( "do not support non-literal expr in transactional insert operation: " + expr.toSql()); } - if (expr instanceof NullLiteral) { - row.addColBuilder().setValue(NULL_VALUE_FOR_LOAD); - } else if (expr instanceof ArrayLiteral) { - row.addColBuilder().setValue("\"" + expr.getStringValueForStreamLoad(options) + "\""); - } else { - String stringValue = expr.getStringValueForStreamLoad(options); - if (stringValue.equals(NULL_VALUE_FOR_LOAD) || stringValue.startsWith("\"") || stringValue.endsWith( - "\"")) { - row.addColBuilder().setValue("\"" + stringValue + "\""); - } else { - row.addColBuilder().setValue(stringValue); - } - } + row.addColBuilder().setValue(expr.getStringValueForStreamLoad(options)); } return row.build(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java index 148bfbf80fa353..2085a407388cce 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java @@ -26,7 +26,7 @@ public class ArrayLiteralTest { @Test - public void testGetStringInFe() throws AnalysisException { + public void testGetStringForQuery() throws AnalysisException { FormatOptions options = FormatOptions.getDefault(); IntLiteral intLiteral1 = new IntLiteral(1); FloatLiteral floatLiteral = new FloatLiteral("2.15"); @@ -40,65 +40,65 @@ public void testGetStringInFe() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueInFe(options)); + Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueForQuery(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); - Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueInFe(options)); + Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueForQuery(options)); ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); - Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueInFe(options)); + Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueForQuery(options)); ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueInFe(options)); + Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueForQuery(options)); ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", - arrayLiteral4.getStringValueInFe(options)); + arrayLiteral4.getStringValueForQuery(options)); ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueInFe(options)); + Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueForQuery(options)); ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueInFe(options)); + Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueForQuery(options)); ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]", - arrayLiteral7.getStringValueInFe(options)); + arrayLiteral7.getStringValueForQuery(options)); ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); Assert.assertEquals("[[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"], [\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]]", - arrayLiteral8.getStringValueInFe(options)); + arrayLiteral8.getStringValueForQuery(options)); ArrayLiteral arrayLiteral9 = new ArrayLiteral(); - Assert.assertEquals("[]", arrayLiteral9.getStringValueInFe(options)); + Assert.assertEquals("[]", arrayLiteral9.getStringValueForQuery(options)); DecimalLiteral decimalLiteral = new DecimalLiteral("1.0"); DecimalLiteral decimalLiteral2 = new DecimalLiteral("2"); ArrayLiteral arrayLiteral10 = new ArrayLiteral(decimalLiteral, decimalLiteral2); - Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueInFe(options)); + Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueForQuery(options)); //array(1, null) IntLiteral intLiteralWithNull = new IntLiteral(1); ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); - Assert.assertEquals("[1, null]", arrayLiteral11.getStringValueInFe(options)); + Assert.assertEquals("[1, null]", arrayLiteral11.getStringValueForQuery(options)); //array(null, 1) ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); - Assert.assertEquals("[null, 1]", arrayLiteral12.getStringValueInFe(options)); + Assert.assertEquals("[null, 1]", arrayLiteral12.getStringValueForQuery(options)); ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", - arrayLiteral13.getStringValueInFe(options)); + arrayLiteral13.getStringValueForQuery(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueInFe(options)); + Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueForQuery(options)); ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); Assert.assertEquals("[{\"col1\":1, \"col2\":2.15, \"col3\":\"2022-10-10\"}]", - arrayLiteral15.getStringValueInFe(options)); + arrayLiteral15.getStringValueForQuery(options)); } @Test - public void testGetStringInFeForPresto() throws AnalysisException { + public void testGetStringForQueryForPresto() throws AnalysisException { FormatOptions options = FormatOptions.getForPresto(); IntLiteral intLiteral1 = new IntLiteral(1); FloatLiteral floatLiteral = new FloatLiteral("2.15"); @@ -112,63 +112,63 @@ public void testGetStringInFeForPresto() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueInFe(options)); + Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueForQuery(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); - Assert.assertEquals("[11:22:33]", arrayLiteralWithTime.getStringValueInFe(options)); + Assert.assertEquals("[11:22:33]", arrayLiteralWithTime.getStringValueForQuery(options)); ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); - Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueInFe(options)); + Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueForQuery(options)); ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[shortstring, shortstring]", arrayLiteral3.getStringValueInFe(options)); + Assert.assertEquals("[shortstring, shortstring]", arrayLiteral3.getStringValueForQuery(options)); ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", - arrayLiteral4.getStringValueInFe(options)); + arrayLiteral4.getStringValueForQuery(options)); ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[NULL, NULL]", arrayLiteral5.getStringValueInFe(options)); + Assert.assertEquals("[NULL, NULL]", arrayLiteral5.getStringValueForQuery(options)); ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[2022-10-10, 2022-10-10]", arrayLiteral6.getStringValueInFe(options)); + Assert.assertEquals("[2022-10-10, 2022-10-10]", arrayLiteral6.getStringValueForQuery(options)); ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); Assert.assertEquals("[2022-10-10 12:10:10, 2022-10-10 12:10:10]", - arrayLiteral7.getStringValueInFe(options)); + arrayLiteral7.getStringValueForQuery(options)); ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); Assert.assertEquals("[[2022-10-10 12:10:10, 2022-10-10 12:10:10], [2022-10-10 12:10:10, 2022-10-10 12:10:10]]", - arrayLiteral8.getStringValueInFe(options)); + arrayLiteral8.getStringValueForQuery(options)); ArrayLiteral arrayLiteral9 = new ArrayLiteral(); - Assert.assertEquals("[]", arrayLiteral9.getStringValueInFe(options)); + Assert.assertEquals("[]", arrayLiteral9.getStringValueForQuery(options)); DecimalLiteral decimalLiteral = new DecimalLiteral("1.0"); DecimalLiteral decimalLiteral2 = new DecimalLiteral("2"); ArrayLiteral arrayLiteral10 = new ArrayLiteral(decimalLiteral, decimalLiteral2); - Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueInFe(options)); + Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueForQuery(options)); //array(1, null) IntLiteral intLiteralWithNull = new IntLiteral(1); ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); - Assert.assertEquals("[1, NULL]", arrayLiteral11.getStringValueInFe(options)); + Assert.assertEquals("[1, NULL]", arrayLiteral11.getStringValueForQuery(options)); //array(null, 1) ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); - Assert.assertEquals("[NULL, 1]", arrayLiteral12.getStringValueInFe(options)); + Assert.assertEquals("[NULL, 1]", arrayLiteral12.getStringValueForQuery(options)); ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", arrayLiteral13.getStringValueInFe(options)); + Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", arrayLiteral13.getStringValueForQuery(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{1=2.15}]", arrayLiteral14.getStringValueInFe(options)); + Assert.assertEquals("[{1=2.15}]", arrayLiteral14.getStringValueForQuery(options)); ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); - Assert.assertEquals("[{col1=1, col2=2.15, col3=2022-10-10}]", arrayLiteral15.getStringValueInFe(options)); + Assert.assertEquals("[{col1=1, col2=2.15, col3=2022-10-10}]", arrayLiteral15.getStringValueForQuery(options)); } @Test - public void testGetStringInFeForHive() throws AnalysisException { + public void testGetStringForQueryForHive() throws AnalysisException { FormatOptions options = FormatOptions.getForHive(); IntLiteral intLiteral1 = new IntLiteral(1); FloatLiteral floatLiteral = new FloatLiteral("2.15"); @@ -182,60 +182,133 @@ public void testGetStringInFeForHive() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1.0,2.15]", arrayLiteral1.getStringValueInFe(options)); + Assert.assertEquals("[1.0,2.15]", arrayLiteral1.getStringValueForQuery(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); - Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueInFe(options)); + Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueForQuery(options)); ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); - Assert.assertEquals("[true,true]", arrayLiteral2.getStringValueInFe(options)); + Assert.assertEquals("[true,true]", arrayLiteral2.getStringValueForQuery(options)); ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[\"shortstring\",\"shortstring\"]", arrayLiteral3.getStringValueInFe(options)); + Assert.assertEquals("[\"shortstring\",\"shortstring\"]", arrayLiteral3.getStringValueForQuery(options)); ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); Assert.assertEquals("[1000000000000000000000,1000000000000000000000]", - arrayLiteral4.getStringValueInFe(options)); + arrayLiteral4.getStringValueForQuery(options)); ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[null,null]", arrayLiteral5.getStringValueInFe(options)); + Assert.assertEquals("[null,null]", arrayLiteral5.getStringValueForQuery(options)); ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[\"2022-10-10\",\"2022-10-10\"]", arrayLiteral6.getStringValueInFe(options)); + Assert.assertEquals("[\"2022-10-10\",\"2022-10-10\"]", arrayLiteral6.getStringValueForQuery(options)); ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); Assert.assertEquals("[\"2022-10-10 12:10:10\",\"2022-10-10 12:10:10\"]", - arrayLiteral7.getStringValueInFe(options)); + arrayLiteral7.getStringValueForQuery(options)); ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); Assert.assertEquals( "[[\"2022-10-10 12:10:10\",\"2022-10-10 12:10:10\"],[\"2022-10-10 12:10:10\",\"2022-10-10 12:10:10\"]]", - arrayLiteral8.getStringValueInFe(options)); + arrayLiteral8.getStringValueForQuery(options)); ArrayLiteral arrayLiteral9 = new ArrayLiteral(); - Assert.assertEquals("[]", arrayLiteral9.getStringValueInFe(options)); + Assert.assertEquals("[]", arrayLiteral9.getStringValueForQuery(options)); DecimalLiteral decimalLiteral = new DecimalLiteral("1.0"); DecimalLiteral decimalLiteral2 = new DecimalLiteral("2"); ArrayLiteral arrayLiteral10 = new ArrayLiteral(decimalLiteral, decimalLiteral2); - Assert.assertEquals("[1.0,2.0]", arrayLiteral10.getStringValueInFe(options)); + Assert.assertEquals("[1.0,2.0]", arrayLiteral10.getStringValueForQuery(options)); //array(1, null) IntLiteral intLiteralWithNull = new IntLiteral(1); ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); - Assert.assertEquals("[1,null]", arrayLiteral11.getStringValueInFe(options)); + Assert.assertEquals("[1,null]", arrayLiteral11.getStringValueForQuery(options)); //array(null, 1) ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); - Assert.assertEquals("[null,1]", arrayLiteral12.getStringValueInFe(options)); + Assert.assertEquals("[null,1]", arrayLiteral12.getStringValueForQuery(options)); ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[1.0,2.15],[1.0,2.15]]", arrayLiteral13.getStringValueInFe(options)); + Assert.assertEquals("[[1.0,2.15],[1.0,2.15]]", arrayLiteral13.getStringValueForQuery(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueInFe(options)); + Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueForQuery(options)); ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); Assert.assertEquals("[{\"col1\":1,\"col2\":2.15,\"col3\":\"2022-10-10\"}]", - arrayLiteral15.getStringValueInFe(options)); + arrayLiteral15.getStringValueForQuery(options)); + } + + @Test + public void testGetStringForStreamLoad() throws AnalysisException { + FormatOptions options = FormatOptions.getDefault(); + IntLiteral intLiteral1 = new IntLiteral(1); + FloatLiteral floatLiteral = new FloatLiteral("2.15"); + FloatLiteral floatLiteral1 = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), + FloatLiteral.getDefaultTimeType(Type.TIME)); + + BoolLiteral boolLiteral = new BoolLiteral(true); + StringLiteral stringLiteral = new StringLiteral("shortstring"); + LargeIntLiteral largeIntLiteral = new LargeIntLiteral("1000000000000000000000"); + NullLiteral nullLiteral = new NullLiteral(); + DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); + DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); + ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); + Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueForStreamLoad(options)); + ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); + Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueForStreamLoad(options)); + + ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); + Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueForStreamLoad(options)); + + ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); + Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueForStreamLoad(options)); + + ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); + Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", + arrayLiteral4.getStringValueForStreamLoad(options)); + + ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); + Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueForStreamLoad(options)); + + ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); + Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueForStreamLoad(options)); + + ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); + Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]", + arrayLiteral7.getStringValueForStreamLoad(options)); + + ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); + Assert.assertEquals( + "[[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"], [\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]]", + arrayLiteral8.getStringValueForStreamLoad(options)); + + ArrayLiteral arrayLiteral9 = new ArrayLiteral(); + Assert.assertEquals("[]", arrayLiteral9.getStringValueForStreamLoad(options)); + + DecimalLiteral decimalLiteral = new DecimalLiteral("1.0"); + DecimalLiteral decimalLiteral2 = new DecimalLiteral("2"); + ArrayLiteral arrayLiteral10 = new ArrayLiteral(decimalLiteral, decimalLiteral2); + Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueForStreamLoad(options)); + + //array(1, null) + IntLiteral intLiteralWithNull = new IntLiteral(1); + ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); + Assert.assertEquals("[1, null]", arrayLiteral11.getStringValueForStreamLoad(options)); + //array(null, 1) + ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); + Assert.assertEquals("[null, 1]", arrayLiteral12.getStringValueForStreamLoad(options)); + + ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); + MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); + StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); + ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); + Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", + arrayLiteral13.getStringValueForStreamLoad(options)); + ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); + Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueForStreamLoad(options)); + ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); + Assert.assertEquals("[{\"col1\":1, \"col2\":2.15, \"col3\":\"2022-10-10\"}]", + arrayLiteral15.getStringValueForStreamLoad(options)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java index 57357952bc68f6..1fb0f78e420a55 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java @@ -35,9 +35,9 @@ public class DateLiteralTest { @Test public void testGetStringInFe() throws AnalysisException { DateLiteral literal = new DateLiteral("1997-10-07", Type.DATE); - String s = literal.getStringValueInFe(FormatOptions.getDefault()); + String s = literal.getStringValueForQuery(FormatOptions.getDefault()); Assert.assertEquals(s, "1997-10-07"); - Assert.assertEquals(literal.getStringValueInFe(FormatOptions.getForPresto()), "1997-10-07"); + Assert.assertEquals(literal.getStringValueForQuery(FormatOptions.getForPresto()), "1997-10-07"); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/DecimalLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/DecimalLiteralTest.java index dd6213c330f6bb..4dfcb789fc5700 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/DecimalLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DecimalLiteralTest.java @@ -36,9 +36,10 @@ public class DecimalLiteralTest { public void testGetStringInFe() { BigDecimal decimal = new BigDecimal("-123456789123456789.123456789"); DecimalLiteral literal = new DecimalLiteral(decimal); - String s = literal.getStringValueInFe(FormatOptions.getDefault()); + String s = literal.getStringValueForQuery(FormatOptions.getDefault()); Assert.assertEquals("-123456789123456789.123456789", s); - Assert.assertEquals("-123456789123456789.123456789", literal.getStringValueInFe(FormatOptions.getForPresto())); + Assert.assertEquals("-123456789123456789.123456789", + literal.getStringValueForQuery(FormatOptions.getForPresto())); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/FloatLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/FloatLiteralTest.java index 63fbea30bf7ee8..6dd4113e3898e8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/FloatLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/FloatLiteralTest.java @@ -30,13 +30,13 @@ public class FloatLiteralTest { public void testGetStringInFe() throws AnalysisException { FloatLiteral literal = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), FloatLiteral.getDefaultTimeType(Type.TIME)); - String s = literal.getStringValueInFe(FormatOptions.getDefault()); + String s = literal.getStringValueForQuery(FormatOptions.getDefault()); Assert.assertEquals("11:22:33", s); - Assert.assertEquals("11:22:33", literal.getStringValueInFe(FormatOptions.getForPresto())); + Assert.assertEquals("11:22:33", literal.getStringValueForQuery(FormatOptions.getForPresto())); FloatLiteral literal1 = new FloatLiteral(11.22); - String s1 = literal1.getStringValueInFe(FormatOptions.getDefault()); + String s1 = literal1.getStringValueForQuery(FormatOptions.getDefault()); Assert.assertEquals("11.22", s1); - Assert.assertEquals("11.22", literal1.getStringValueInFe(FormatOptions.getForPresto())); + Assert.assertEquals("11.22", literal1.getStringValueForQuery(FormatOptions.getForPresto())); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java index 3c15fe3ad5f9ff..9e483e5177117d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java @@ -61,32 +61,33 @@ public static void setUp() throws AnalysisException { } @Test - public void testGetStringInFe() throws AnalysisException { + public void testGetStringForQuery() throws AnalysisException { FormatOptions options = FormatOptions.getDefault(); MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueInFe(options)); + Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueForQuery(options)); MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); - Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueInFe(options)); + Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueForQuery(options)); MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{1:\"shortstring\"}", mapLiteral2.getStringValueInFe(options)); + Assertions.assertEquals("{1:\"shortstring\"}", mapLiteral2.getStringValueForQuery(options)); MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", mapLiteral3.getStringValueInFe(options)); + Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", mapLiteral3.getStringValueForQuery(options)); MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); - Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueInFe(options)); + Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueForQuery(options)); MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); - Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", mapLiteral5.getStringValueInFe(options)); + Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", + mapLiteral5.getStringValueForQuery(options)); MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); - Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueInFe(options)); + Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueForQuery(options)); MapLiteral mapLiteral7 = new MapLiteral(); - Assertions.assertEquals("{}", mapLiteral7.getStringValueInFe(options)); + Assertions.assertEquals("{}", mapLiteral7.getStringValueForQuery(options)); MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueInFe(options)); + Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueForQuery(options)); MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueInFe(options)); + Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueForQuery(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1:[1.0, 2.15]}", mapLiteral10.getStringValueInFe(options)); + Assertions.assertEquals("{1:[1.0, 2.15]}", mapLiteral10.getStringValueForQuery(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -95,7 +96,7 @@ public void testGetStringInFe() throws AnalysisException { } MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{1.0:{1:2.15}}", mapLiteral12.getStringValueInFe(options)); + Assertions.assertEquals("{1.0:{1:2.15}}", mapLiteral12.getStringValueForQuery(options)); try { new MapLiteral(mapLiteral, decimalLiteral1); } catch (Exception e) { @@ -105,7 +106,7 @@ public void testGetStringInFe() throws AnalysisException { MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); Assertions.assertEquals("{\"shortstring\":{\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", - mapLiteral13.getStringValueInFe(options)); + mapLiteral13.getStringValueForQuery(options)); try { new MapLiteral(structLiteral, stringLiteral); } catch (Exception e) { @@ -116,32 +117,32 @@ public void testGetStringInFe() throws AnalysisException { } @Test - public void testGetStringInFeForPresto() throws AnalysisException { + public void testGetStringForQueryForPresto() throws AnalysisException { FormatOptions options = FormatOptions.getForPresto(); MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{1=2.15}", mapLiteral1.getStringValueInFe(options)); + Assertions.assertEquals("{1=2.15}", mapLiteral1.getStringValueForQuery(options)); MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); - Assertions.assertEquals("{1=11:22:33}", mapLiteral11.getStringValueInFe(options)); + Assertions.assertEquals("{1=11:22:33}", mapLiteral11.getStringValueForQuery(options)); MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{1=shortstring}", mapLiteral2.getStringValueInFe(options)); + Assertions.assertEquals("{1=shortstring}", mapLiteral2.getStringValueForQuery(options)); MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{1000000000000000000000=2022-10-10}", mapLiteral3.getStringValueInFe(options)); + Assertions.assertEquals("{1000000000000000000000=2022-10-10}", mapLiteral3.getStringValueForQuery(options)); MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); - Assertions.assertEquals("{11:22:33=NULL}", mapLiteral4.getStringValueInFe(options)); + Assertions.assertEquals("{11:22:33=NULL}", mapLiteral4.getStringValueForQuery(options)); MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); - Assertions.assertEquals("{2022-10-10 12:10:10=2022-10-10}", mapLiteral5.getStringValueInFe(options)); + Assertions.assertEquals("{2022-10-10 12:10:10=2022-10-10}", mapLiteral5.getStringValueForQuery(options)); MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); - Assertions.assertEquals("{1.0=2}", mapLiteral6.getStringValueInFe(options)); + Assertions.assertEquals("{1.0=2}", mapLiteral6.getStringValueForQuery(options)); MapLiteral mapLiteral7 = new MapLiteral(); - Assertions.assertEquals("{}", mapLiteral7.getStringValueInFe(options)); + Assertions.assertEquals("{}", mapLiteral7.getStringValueForQuery(options)); MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{NULL=1}", mapLiteral8.getStringValueInFe(options)); + Assertions.assertEquals("{NULL=1}", mapLiteral8.getStringValueForQuery(options)); MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{1=NULL}", mapLiteral9.getStringValueInFe(options)); + Assertions.assertEquals("{1=NULL}", mapLiteral9.getStringValueForQuery(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1=[1.0, 2.15]}", mapLiteral10.getStringValueInFe(options)); + Assertions.assertEquals("{1=[1.0, 2.15]}", mapLiteral10.getStringValueForQuery(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -150,7 +151,7 @@ public void testGetStringInFeForPresto() throws AnalysisException { } MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{1.0={1=2.15}}", mapLiteral12.getStringValueInFe(options)); + Assertions.assertEquals("{1.0={1=2.15}}", mapLiteral12.getStringValueForQuery(options)); try { new MapLiteral(mapLiteral, decimalLiteral1); } catch (Exception e) { @@ -160,7 +161,7 @@ public void testGetStringInFeForPresto() throws AnalysisException { MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); Assertions.assertEquals("{shortstring={col1=1, col2=2.15, col3=1.0, col4=2022-10-10}}", - mapLiteral13.getStringValueInFe(options)); + mapLiteral13.getStringValueForQuery(options)); try { new MapLiteral(structLiteral, stringLiteral); } catch (Exception e) { @@ -171,32 +172,33 @@ public void testGetStringInFeForPresto() throws AnalysisException { } @Test - public void testGetStringInFeForHive() throws AnalysisException { + public void testGetStringForQueryForHive() throws AnalysisException { FormatOptions options = FormatOptions.getForHive(); MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueInFe(options)); + Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueForQuery(options)); MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); - Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueInFe(options)); + Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueForQuery(options)); MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{true:\"shortstring\"}", mapLiteral2.getStringValueInFe(options)); + Assertions.assertEquals("{true:\"shortstring\"}", mapLiteral2.getStringValueForQuery(options)); MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", mapLiteral3.getStringValueInFe(options)); + Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", mapLiteral3.getStringValueForQuery(options)); MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); - Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueInFe(options)); + Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueForQuery(options)); MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); - Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", mapLiteral5.getStringValueInFe(options)); + Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", + mapLiteral5.getStringValueForQuery(options)); MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); - Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueInFe(options)); + Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueForQuery(options)); MapLiteral mapLiteral7 = new MapLiteral(); - Assertions.assertEquals("{}", mapLiteral7.getStringValueInFe(options)); + Assertions.assertEquals("{}", mapLiteral7.getStringValueForQuery(options)); MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueInFe(options)); + Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueForQuery(options)); MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueInFe(options)); + Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueForQuery(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1:[1.0,2.15]}", mapLiteral10.getStringValueInFe(options)); + Assertions.assertEquals("{1:[1.0,2.15]}", mapLiteral10.getStringValueForQuery(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -205,7 +207,7 @@ public void testGetStringInFeForHive() throws AnalysisException { } MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{1.0:{1:2.15}}", mapLiteral12.getStringValueInFe(options)); + Assertions.assertEquals("{1.0:{1:2.15}}", mapLiteral12.getStringValueForQuery(options)); try { new MapLiteral(mapLiteral, decimalLiteral1); } catch (Exception e) { @@ -215,7 +217,64 @@ public void testGetStringInFeForHive() throws AnalysisException { MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); Assertions.assertEquals("{\"shortstring\":{\"col1\":1,\"col2\":2.15,\"col3\":1.0,\"col4\":\"2022-10-10\"}}", - mapLiteral13.getStringValueInFe(options)); + mapLiteral13.getStringValueForQuery(options)); + try { + new MapLiteral(structLiteral, stringLiteral); + } catch (Exception e) { + Assertions.assertEquals("errCode = 2, " + + "detailMessage = Invalid key type in Map, " + + "not support struct", e.getMessage()); + } + } + + @Test + public void testGetStringForStreamLoad() throws AnalysisException { + FormatOptions options = FormatOptions.getDefault(); + MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); + Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueForStreamLoad(options)); + MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); + Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueForStreamLoad(options)); + MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); + Assertions.assertEquals("{1:\"shortstring\"}", mapLiteral2.getStringValueForStreamLoad(options)); + MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); + Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", + mapLiteral3.getStringValueForStreamLoad(options)); + MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); + Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueForStreamLoad(options)); + MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); + Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", + mapLiteral5.getStringValueForStreamLoad(options)); + MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); + Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueForStreamLoad(options)); + + MapLiteral mapLiteral7 = new MapLiteral(); + Assertions.assertEquals("{}", mapLiteral7.getStringValueForStreamLoad(options)); + MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); + Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueForStreamLoad(options)); + MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); + Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueForStreamLoad(options)); + + MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); + Assertions.assertEquals("{1:[1.0, 2.15]}", mapLiteral10.getStringValueForStreamLoad(options)); + try { + new MapLiteral(arrayLiteral, floatLiteral); + } catch (Exception e) { + Assertions.assertEquals("errCode = 2, " + + "detailMessage = Invalid key type in Map, not support array", e.getMessage()); + } + + MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); + Assertions.assertEquals("{1.0:{1:2.15}}", mapLiteral12.getStringValueForStreamLoad(options)); + try { + new MapLiteral(mapLiteral, decimalLiteral1); + } catch (Exception e) { + Assertions.assertEquals("errCode = 2, " + + "detailMessage = Invalid key type in Map, not support map", e.getMessage()); + } + + MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); + Assertions.assertEquals("{\"shortstring\":{\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", + mapLiteral13.getStringValueForStreamLoad(options)); try { new MapLiteral(structLiteral, stringLiteral); } catch (Exception e) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java index 07ef7363f80ccf..5f7e24bd4ade58 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java @@ -67,16 +67,16 @@ public void testGetStringInFe() throws AnalysisException { stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, datetimeLiteral); Assert.assertEquals("{\"col1\":1, \"col2\":2.15, \"col3\":\"11:22:33\", \"col4\":1, \"col5\":" + "\"shortstring\", \"col6\":1000000000000000000000, \"col7\":1.0, \"col8\":2, \"col9\":\"2022-10-10\", \"col10\":\"2022-10-10 12:10:10\"}", - structLiteral1.getStringValueInFe(options)); + structLiteral1.getStringValueForQuery(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); Assert.assertEquals("{\"col1\":[1.0, 2.15], \"col2\":{1:2.15}, \"col3\":" + "{\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", - structLiteral2.getStringValueInFe(options)); + structLiteral2.getStringValueForQuery(options)); StructLiteral structLiteral3 = new StructLiteral(); - Assert.assertEquals("{}", structLiteral3.getStringValueInFe(options)); + Assert.assertEquals("{}", structLiteral3.getStringValueForQuery(options)); StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); - Assert.assertEquals("{\"col1\":null, \"col2\":1}", nullStruct.getStringValueInFe(options)); + Assert.assertEquals("{\"col1\":null, \"col2\":1}", nullStruct.getStringValueForQuery(options)); } @Test @@ -86,16 +86,16 @@ public void testGetStringInFeForPresto() throws AnalysisException { stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, datetimeLiteral); Assert.assertEquals("{col1=1, col2=2.15, col3=11:22:33, col4=1, col5=" + "shortstring, col6=1000000000000000000000, col7=1.0, col8=2, col9=2022-10-10, col10=2022-10-10 12:10:10}", - structLiteral1.getStringValueInFe(options)); + structLiteral1.getStringValueForQuery(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); Assert.assertEquals("{col1=[1.0, 2.15], col2={1=2.15}, col3=" + "{col1=1, col2=2.15, col3=1.0, col4=2022-10-10}}", - structLiteral2.getStringValueInFe(options)); + structLiteral2.getStringValueForQuery(options)); StructLiteral structLiteral3 = new StructLiteral(); - Assert.assertEquals("{}", structLiteral3.getStringValueInFe(options)); + Assert.assertEquals("{}", structLiteral3.getStringValueForQuery(options)); StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); - Assert.assertEquals("{col1=NULL, col2=1}", nullStruct.getStringValueInFe(options)); + Assert.assertEquals("{col1=NULL, col2=1}", nullStruct.getStringValueForQuery(options)); } @Test @@ -105,15 +105,34 @@ public void testGetStringInFeForHive() throws AnalysisException { stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, datetimeLiteral); Assert.assertEquals( "{\"col1\":1,\"col2\":2.15,\"col3\":\"11:22:33\",\"col4\":true,\"col5\":\"shortstring\",\"col6\":1000000000000000000000,\"col7\":1.0,\"col8\":2,\"col9\":\"2022-10-10\",\"col10\":\"2022-10-10 12:10:10\"}", - structLiteral1.getStringValueInFe(options)); + structLiteral1.getStringValueForQuery(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); Assert.assertEquals( "{\"col1\":[1.0,2.15],\"col2\":{1:2.15},\"col3\":{\"col1\":1,\"col2\":2.15,\"col3\":1.0,\"col4\":\"2022-10-10\"}}", - structLiteral2.getStringValueInFe(options)); + structLiteral2.getStringValueForQuery(options)); StructLiteral structLiteral3 = new StructLiteral(); - Assert.assertEquals("{}", structLiteral3.getStringValueInFe(options)); + Assert.assertEquals("{}", structLiteral3.getStringValueForQuery(options)); StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); - Assert.assertEquals("{\"col1\":null,\"col2\":1}", nullStruct.getStringValueInFe(options)); + Assert.assertEquals("{\"col1\":null,\"col2\":1}", nullStruct.getStringValueForQuery(options)); + } + + @Test + public void testGetStringForStreamLoad() throws AnalysisException { + FormatOptions options = FormatOptions.getDefault(); + StructLiteral structLiteral1 = new StructLiteral(intLiteral1, floatLiteral, floatLiteral1, boolLiteral, + stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, datetimeLiteral); + Assert.assertEquals( + "{1, 2.15, \"11:22:33\", 1, \"shortstring\", 1000000000000000000000, 1.0, 2, \"2022-10-10\", \"2022-10-10 12:10:10\"}", + structLiteral1.getStringValueForStreamLoad(options)); + StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); + Assert.assertEquals( + "{[1.0, 2.15], {1:2.15}, {\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", + structLiteral2.getStringValueForStreamLoad(options)); + StructLiteral structLiteral3 = new StructLiteral(); + Assert.assertEquals("{}", structLiteral3.getStringValueForStreamLoad(options)); + + StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); + Assert.assertEquals("{null, 1}", nullStruct.getStringValueForStreamLoad(options)); } } From a4fe8a58b039324116803bd5be5c91b9a17d183e Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 10 Apr 2025 17:04:37 -0700 Subject: [PATCH 06/10] 5 --- .../apache/doris/analysis/ArrayLiteral.java | 2 +- .../java/org/apache/doris/analysis/Expr.java | 10 -- .../apache/doris/analysis/FloatLiteral.java | 9 -- .../org/apache/doris/analysis/MapLiteral.java | 4 +- .../apache/doris/analysis/StructLiteral.java | 2 +- .../doris/analysis/ScalarLiteralTest.java | 127 ++++++++++++++++++ 6 files changed, 131 insertions(+), 23 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/ScalarLiteralTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java index 06f5e9fe2e8fea..eb957eb03bb392 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java @@ -145,7 +145,7 @@ public String getStringValueForQuery(FormatOptions options) { public String getStringValueForStreamLoad(FormatOptions options) { List list = new ArrayList<>(children.size()); children.forEach(v -> { - list.add(v.getStringValueInComplexTypeForStreamLoad(options)); + list.add(v.getStringValueInComplexTypeForQuery(options)); }); return "[" + StringUtils.join(list, options.getCollectionDelim()) + "]"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index 0ca9da5904fb22..a7cfad25d465fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -2214,16 +2214,6 @@ public String getStringValueForStreamLoad(FormatOptions options) { return getStringValueForQuery(options); } - /** - * This method is to return the string value of this expr in a complex type for stream load - * It is only used for "getStringValueForStreamLoad()" - * There is a little different from "getStringValueInComplexTypeForQuery()", - * eg, for StructLiteral, the value should not contain sub column's name. - */ - protected String getStringValueInComplexTypeForStreamLoad(FormatOptions options) { - return getStringValueInComplexTypeForQuery(options); - } - public final TExpr normalize(Normalizer normalizer) { TExpr result = new TExpr(); treeToThriftHelper(result, (expr, texprNode) -> expr.normalize(texprNode, normalizer)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java index 732f7eb14055fb..9b2c78dc54ab5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java @@ -178,15 +178,6 @@ public String getStringValueInComplexTypeForQuery(FormatOptions options) { return ret; } - @Override - public String getStringValueForStreamLoad(FormatOptions options) { - String ret = this.getStringValueForQuery(options); - if (type == Type.TIME || type == Type.TIMEV2) { - ret = options.getNestedStringWrapper() + ret + options.getNestedStringWrapper(); - } - return ret; - } - public static Type getDefaultTimeType(Type type) throws AnalysisException { switch (type.getPrimitiveType()) { case TIME: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java index 0031777b428722..3891d3cc337229 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java @@ -194,8 +194,8 @@ public String getStringValueForStreamLoad(FormatOptions options) { // map key type do not support complex type throw new UnsupportedOperationException("Unsupported key type for MAP: " + children.get(i).getType()); } - list.add(children.get(i).getStringValueInComplexTypeForStreamLoad(options) - + options.getMapKeyDelim() + children.get(i + 1).getStringValueInComplexTypeForStreamLoad(options)); + list.add(children.get(i).getStringValueInComplexTypeForQuery(options) + + options.getMapKeyDelim() + children.get(i + 1).getStringValueInComplexTypeForQuery(options)); } return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java index 4c0f16b18e84ee..cc9b7b44b6da35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java @@ -127,7 +127,7 @@ public String getStringValueForStreamLoad(FormatOptions options) { // same with be default field index start with 1 for (int i = 0; i < children.size(); i++) { Expr child = children.get(i); - list.add(child.getStringValueInComplexTypeForStreamLoad(options)); + list.add(child.getStringValueInComplexTypeForQuery(options)); } return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ScalarLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ScalarLiteralTest.java new file mode 100644 index 00000000000000..55f94050d6f62e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ScalarLiteralTest.java @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.FormatOptions; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class ScalarLiteralTest { + static IntLiteral intLiteral1; + static FloatLiteral floatLiteral; + static FloatLiteral floatLiteral1; + static BoolLiteral boolLiteral; + static StringLiteral stringLiteral; + static LargeIntLiteral largeIntLiteral; + static NullLiteral nullLiteral; + static DateLiteral dateLiteral; + static DateLiteral datetimeLiteral; + static DecimalLiteral decimalLiteral1; + static DecimalLiteral decimalLiteral2; + static ArrayLiteral arrayLiteral; + static MapLiteral mapLiteral; + static StructLiteral structLiteral; + + @BeforeAll + public static void setUp() throws AnalysisException { + intLiteral1 = new IntLiteral(1); + floatLiteral = new FloatLiteral("2.15"); + floatLiteral1 = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), + FloatLiteral.getDefaultTimeType(Type.TIME)); + boolLiteral = new BoolLiteral(true); + stringLiteral = new StringLiteral("shortstring"); + largeIntLiteral = new LargeIntLiteral("1000000000000000000000"); + nullLiteral = new NullLiteral(); + dateLiteral = new DateLiteral("2022-10-10", Type.DATE); + datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); + decimalLiteral1 = new DecimalLiteral("1.0"); + decimalLiteral2 = new DecimalLiteral("2"); + arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); + mapLiteral = new MapLiteral(intLiteral1, floatLiteral); + structLiteral = new StructLiteral(intLiteral1, floatLiteral, decimalLiteral1, dateLiteral); + } + + @Test + public void testGetStringForQuery() { + FormatOptions options = FormatOptions.getDefault(); + Assertions.assertEquals("1", intLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2.15", floatLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("11:22:33", floatLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("1", boolLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("shortstring", stringLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1000000000000000000000", largeIntLiteral.getStringValueForQuery(options)); + Assertions.assertEquals(null, nullLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10", dateLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10 12:10:10", datetimeLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1.0", decimalLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2", decimalLiteral2.getStringValueForQuery(options)); + } + + @Test + public void testGetStringForQueryForPresto() { + FormatOptions options = FormatOptions.getForPresto(); + Assertions.assertEquals("1", intLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2.15", floatLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("11:22:33", floatLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("1", boolLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("shortstring", stringLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1000000000000000000000", largeIntLiteral.getStringValueForQuery(options)); + Assertions.assertEquals(null, nullLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10", dateLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10 12:10:10", datetimeLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1.0", decimalLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2", decimalLiteral2.getStringValueForQuery(options)); + } + + @Test + public void testGetStringForQueryForHive() { + FormatOptions options = FormatOptions.getForHive(); + Assertions.assertEquals("1", intLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2.15", floatLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("11:22:33", floatLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("1", boolLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("shortstring", stringLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1000000000000000000000", largeIntLiteral.getStringValueForQuery(options)); + Assertions.assertEquals(null, nullLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10", dateLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10 12:10:10", datetimeLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1.0", decimalLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2", decimalLiteral2.getStringValueForQuery(options)); + } + + @Test + public void testGetStringForStreamLoad() { + FormatOptions options = FormatOptions.getDefault(); + Assertions.assertEquals("1", intLiteral1.getStringValueForStreamLoad(options)); + Assertions.assertEquals("2.15", floatLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("11:22:33", floatLiteral1.getStringValueForStreamLoad(options)); + Assertions.assertEquals("1", boolLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("shortstring", stringLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("1000000000000000000000", largeIntLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals(FeConstants.null_string, nullLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("2022-10-10", dateLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("2022-10-10 12:10:10", datetimeLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("1.0", decimalLiteral1.getStringValueForStreamLoad(options)); + Assertions.assertEquals("2", decimalLiteral2.getStringValueForStreamLoad(options)); + } +} From dedecb0574e0d422471db52ab1925fdf97d956c2 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 10 Apr 2025 17:20:38 -0700 Subject: [PATCH 07/10] 6 --- .../java/org/apache/doris/analysis/ArrayLiteral.java | 2 ++ .../java/org/apache/doris/analysis/BoolLiteral.java | 7 +++++-- .../java/org/apache/doris/analysis/MapLiteral.java | 2 ++ .../org/apache/doris/analysis/StructLiteral.java | 2 ++ .../java/org/apache/doris/common/FormatOptions.java | 2 ++ .../org/apache/doris/analysis/StructLiteralTest.java | 12 ++++++++++++ 6 files changed, 25 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java index eb957eb03bb392..3c55eced008cb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java @@ -135,9 +135,11 @@ public String getStringValue() { @Override public String getStringValueForQuery(FormatOptions options) { List list = new ArrayList<>(children.size()); + ++options.level; children.forEach(v -> { list.add(v.getStringValueInComplexTypeForQuery(options)); }); + --options.level; return "[" + StringUtils.join(list, options.getCollectionDelim()) + "]"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java index 39d91253692f96..7008872ababc4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java @@ -114,8 +114,11 @@ public String getStringValue() { @Override public String getStringValueForQuery(FormatOptions options) { - String val = options.isBoolValueNum() ? getStringValue() : (value ? "true" : "false"); - return val; + if (options.level > 0) { + return options.isBoolValueNum() ? getStringValue() : (value ? "true" : "false"); + } else { + return getStringValue(); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java index 3891d3cc337229..a5ec0f2cbf755a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java @@ -173,6 +173,7 @@ private String getStringValue(Expr expr) { @Override public String getStringValueForQuery(FormatOptions options) { List list = new ArrayList<>(children.size()); + ++options.level; for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) { // we should use type to decide we output array is suitable for json format if (children.get(i).getType().isComplexType()) { @@ -182,6 +183,7 @@ public String getStringValueForQuery(FormatOptions options) { list.add(children.get(i).getStringValueInComplexTypeForQuery(options) + options.getMapKeyDelim() + children.get(i + 1).getStringValueInComplexTypeForQuery(options)); } + --options.level; return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java index cc9b7b44b6da35..78437c26ce07cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java @@ -109,6 +109,7 @@ public String getStringValue() { @Override public String getStringValueForQuery(FormatOptions options) { List list = new ArrayList<>(children.size()); + ++options.level; // same with be default field index start with 1 for (int i = 0; i < children.size(); i++) { Expr child = children.get(i); @@ -118,6 +119,7 @@ public String getStringValueForQuery(FormatOptions options) { + options.getMapKeyDelim() + child.getStringValueInComplexTypeForQuery(options)); } + --options.level; return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java b/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java index fc7e0f4726b719..39d939a6a8eeda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java @@ -35,6 +35,8 @@ public class FormatOptions { // This is only for boolean column within the collection type. // For top level boolean column, it is always 0/1 private boolean isBoolValueNum; + // Indicate the nested level of column. It is used to control some behavior of serde + public int level = 0; private FormatOptions(String nestedStringWrapper, String mapKeyDelim, String nullFormat, String collectionDelim, boolean isBoolValueNum) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java index 5f7e24bd4ade58..5df4dc38e88021 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java @@ -77,6 +77,9 @@ public void testGetStringInFe() throws AnalysisException { StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); Assert.assertEquals("{\"col1\":null, \"col2\":1}", nullStruct.getStringValueForQuery(options)); + + StructLiteral boolStruct = new StructLiteral(boolLiteral, intLiteral1); + Assert.assertEquals("{\"col1\":1, \"col2\":1}", boolStruct.getStringValueForQuery(options)); } @Test @@ -96,6 +99,9 @@ public void testGetStringInFeForPresto() throws AnalysisException { StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); Assert.assertEquals("{col1=NULL, col2=1}", nullStruct.getStringValueForQuery(options)); + + StructLiteral boolStruct = new StructLiteral(boolLiteral, intLiteral1); + Assert.assertEquals("{col1=1, col2=1}", boolStruct.getStringValueForQuery(options)); } @Test @@ -115,6 +121,9 @@ public void testGetStringInFeForHive() throws AnalysisException { StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); Assert.assertEquals("{\"col1\":null,\"col2\":1}", nullStruct.getStringValueForQuery(options)); + + StructLiteral boolStruct = new StructLiteral(boolLiteral, intLiteral1); + Assert.assertEquals("{\"col1\":true,\"col2\":1}", boolStruct.getStringValueForQuery(options)); } @Test @@ -134,5 +143,8 @@ public void testGetStringForStreamLoad() throws AnalysisException { StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); Assert.assertEquals("{null, 1}", nullStruct.getStringValueForStreamLoad(options)); + + StructLiteral boolStruct = new StructLiteral(boolLiteral, intLiteral1); + Assert.assertEquals("{1, 1}", boolStruct.getStringValueForStreamLoad(options)); } } From 54c7364584e56f1aa2ef328afe7e83c758ffc696 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 11 Apr 2025 11:53:49 -0700 Subject: [PATCH 08/10] 7 --- .../org/apache/doris/analysis/ArrayLiteral.java | 9 --------- .../java/org/apache/doris/analysis/CastExpr.java | 2 +- .../org/apache/doris/analysis/DateLiteral.java | 2 +- .../org/apache/doris/analysis/FloatLiteral.java | 2 +- .../org/apache/doris/analysis/IPv4Literal.java | 2 +- .../org/apache/doris/analysis/IPv6Literal.java | 2 +- .../org/apache/doris/analysis/JsonLiteral.java | 2 +- .../org/apache/doris/analysis/LiteralExpr.java | 5 ----- .../org/apache/doris/analysis/MapLiteral.java | 15 --------------- .../org/apache/doris/analysis/MaxLiteral.java | 2 +- .../org/apache/doris/analysis/NullLiteral.java | 2 +- .../apache/doris/analysis/PlaceHolderExpr.java | 2 +- .../org/apache/doris/analysis/StringLiteral.java | 2 +- 13 files changed, 10 insertions(+), 39 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java index 3c55eced008cb2..b798f2f15bea85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java @@ -143,15 +143,6 @@ public String getStringValueForQuery(FormatOptions options) { return "[" + StringUtils.join(list, options.getCollectionDelim()) + "]"; } - @Override - public String getStringValueForStreamLoad(FormatOptions options) { - List list = new ArrayList<>(children.size()); - children.forEach(v -> { - list.add(v.getStringValueInComplexTypeForQuery(options)); - }); - return "[" + StringUtils.join(list, options.getCollectionDelim()) + "]"; - } - @Override protected void toThrift(TExprNode msg) { msg.node_type = TExprNodeType.ARRAY_LITERAL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index 437c0e5cae45ee..aab6a9dbec0747 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -582,7 +582,7 @@ public String getStringValueForStreamLoad(FormatOptions options) { } @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return children.get(0).getStringValueInComplexTypeForQuery(options); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 69d5ee4a5f201f..e881874c515300 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -765,7 +765,7 @@ public String getStringValue(Type type) { } @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java index 9b2c78dc54ab5e..074fc250967820 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java @@ -170,7 +170,7 @@ public String getStringValueForQuery(FormatOptions options) { } @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { String ret = this.getStringValueForQuery(options); if (type == Type.TIME || type == Type.TIMEV2) { ret = options.getNestedStringWrapper() + ret + options.getNestedStringWrapper(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java index 399e9979b6c1a1..33a273ad52f664 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java @@ -140,7 +140,7 @@ public String getStringValue() { } @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java index 2db22de8eac36c..f33f1103d95e2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java @@ -105,7 +105,7 @@ public String getStringValue() { } @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java index 0a56eff84a0ec0..050ec16a3304e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java @@ -101,7 +101,7 @@ public String getStringValue() { } @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 76ae813574d8b2..6f3ca405d95c2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -243,11 +243,6 @@ public String getStringValueForQuery(FormatOptions options) { return getStringValue(); } - @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { - return getStringValueForQuery(options); - } - public long getLongValue() { return 0; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java index a5ec0f2cbf755a..396883699c5f65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java @@ -187,21 +187,6 @@ public String getStringValueForQuery(FormatOptions options) { return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } - @Override - public String getStringValueForStreamLoad(FormatOptions options) { - List list = new ArrayList<>(children.size()); - for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) { - // we should use type to decide we output array is suitable for json format - if (children.get(i).getType().isComplexType()) { - // map key type do not support complex type - throw new UnsupportedOperationException("Unsupported key type for MAP: " + children.get(i).getType()); - } - list.add(children.get(i).getStringValueInComplexTypeForQuery(options) - + options.getMapKeyDelim() + children.get(i + 1).getStringValueInComplexTypeForQuery(options)); - } - return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; - } - @Override protected String toSqlImpl() { List list = new ArrayList<>(children.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java index ce6dc0100ec63f..03c4653833d5d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java @@ -69,7 +69,7 @@ public String getStringValue() { } @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java index 24ec69523c1a95..70840bb4fdb648 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java @@ -119,7 +119,7 @@ public String getStringValueForStreamLoad(FormatOptions options) { // the null value inside an array is represented as "null", for exampe: // [null, null]. Not same as other primitive type to represent as \N. @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return options.getNullFormat(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java index 646f2c9c7bb2e6..eda712552e86e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java @@ -191,7 +191,7 @@ public Expr reset() { } @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index c475ac1b680df1..51f1bb53b95ebe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -154,7 +154,7 @@ public String getStringValue() { } @Override - public String getStringValueInComplexTypeForQuery(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } From 5c5b0f0ae460eeb21d80e0f88e0b2877fb569000 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 12 Apr 2025 05:35:19 +0800 Subject: [PATCH 09/10] fix batch insert trim quota --- .../doris/nereids/trees/plans/commands/insert/InsertUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 442c846b67efc1..efa363d8d4015e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -244,7 +244,6 @@ private static void beginBatchInsertTransaction(ConnectContext ctx, .setTimeout((int) timeoutSecond) .setTimezone(timeZone) .setSendBatchParallelism(sendBatchParallelism) - .setTrimDoubleQuotes(true) .setSequenceCol(columns.stream() .filter(c -> Column.SEQUENCE_COL.equalsIgnoreCase(c.getName())) .map(Column::getName) From 5943c39f315bc3ac279a7aa1fd1cecc551b414db Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 12 Apr 2025 05:38:14 +0800 Subject: [PATCH 10/10] 9 --- .../data_types/serde/data_type_string_serde.h | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/be/src/vec/data_types/serde/data_type_string_serde.h b/be/src/vec/data_types/serde/data_type_string_serde.h index bd1dac19725d99..4af97453e00671 100644 --- a/be/src/vec/data_types/serde/data_type_string_serde.h +++ b/be/src/vec/data_types/serde/data_type_string_serde.h @@ -169,16 +169,16 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status deserialize_one_cell_from_json(IColumn& column, Slice& slice, const FormatOptions& options) const override { /* - * For strings in the json complex type, we remove double quotes by default. - * - * Because when querying complex types, such as selecting complexColumn from table, - * we will add double quotes to the strings in the complex type. - * - * For the map column, insert { "abc" : 1, "hello",2 }. - * If you do not remove the double quotes, it will display {""abc"":1,""hello"": 2 }, - * remove the double quotes to display { "abc" : 1, "hello",2 }. - * - */ + * For strings in the json complex type, we remove double quotes by default. + * + * Because when querying complex types, such as selecting complexColumn from table, + * we will add double quotes to the strings in the complex type. + * + * For the map column, insert { "abc" : 1, "hello",2 }. + * If you do not remove the double quotes, it will display {""abc"":1,""hello"": 2 }, + * remove the double quotes to display { "abc" : 1, "hello",2 }. + * + */ if (_nesting_level >= 2) { slice.trim_quote(); }