diff --git a/be/src/vec/data_types/serde/data_type_string_serde.h b/be/src/vec/data_types/serde/data_type_string_serde.h index bd1dac19725d99..4af97453e00671 100644 --- a/be/src/vec/data_types/serde/data_type_string_serde.h +++ b/be/src/vec/data_types/serde/data_type_string_serde.h @@ -169,16 +169,16 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status deserialize_one_cell_from_json(IColumn& column, Slice& slice, const FormatOptions& options) const override { /* - * For strings in the json complex type, we remove double quotes by default. - * - * Because when querying complex types, such as selecting complexColumn from table, - * we will add double quotes to the strings in the complex type. - * - * For the map column, insert { "abc" : 1, "hello",2 }. - * If you do not remove the double quotes, it will display {""abc"":1,""hello"": 2 }, - * remove the double quotes to display { "abc" : 1, "hello",2 }. - * - */ + * For strings in the json complex type, we remove double quotes by default. + * + * Because when querying complex types, such as selecting complexColumn from table, + * we will add double quotes to the strings in the complex type. + * + * For the map column, insert { "abc" : 1, "hello",2 }. + * If you do not remove the double quotes, it will display {""abc"":1,""hello"": 2 }, + * remove the double quotes to display { "abc" : 1, "hello",2 }. + * + */ if (_nesting_level >= 2) { slice.trim_quote(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java index 2e0f43758f8e79..b798f2f15bea85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java @@ -133,42 +133,14 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { - List list = new ArrayList<>(children.size()); - children.forEach(v -> list.add(v.getStringValueForArray(options))); - return "[" + StringUtils.join(list, ", ") + "]"; - } - - @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { List list = new ArrayList<>(children.size()); + ++options.level; children.forEach(v -> { - String stringLiteral; - if (v instanceof NullLiteral) { - stringLiteral = options.getNullFormat(); - } else { - stringLiteral = getStringLiteralForComplexType(v, options); - } - // we should use type to decide we output array is suitable for json format - list.add(stringLiteral); + list.add(v.getStringValueInComplexTypeForQuery(options)); }); - return "[" + StringUtils.join(list, ", ") + "]"; - } - - @Override - public String getStringValueForStreamLoad(FormatOptions options) { - List list = new ArrayList<>(children.size()); - children.forEach(v -> { - String stringLiteral; - if (v instanceof NullLiteral) { - stringLiteral = "null"; - } else { - stringLiteral = getStringLiteralForStreamLoad(v, options); - } - // we should use type to decide we output array is suitable for json format - list.add(stringLiteral); - }); - return "[" + StringUtils.join(list, ", ") + "]"; + --options.level; + return "[" + StringUtils.join(list, options.getCollectionDelim()) + "]"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java index c0d6d885285151..7008872ababc4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java @@ -111,9 +111,14 @@ public String getStringValue() { return value ? "1" : "0"; } + @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + public String getStringValueForQuery(FormatOptions options) { + if (options.level > 0) { + return options.isBoolValueNum() ? getStringValue() : (value ? "true" : "false"); + } else { + return getStringValue(); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index 75bc129b523d0a..aab6a9dbec0747 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -577,8 +577,13 @@ public boolean isNullable() { } @Override - public String getStringValueForArray(FormatOptions options) { - return children.get(0).getStringValueForArray(options); + public String getStringValueForStreamLoad(FormatOptions options) { + return children.get(0).getStringValueForStreamLoad(options); + } + + @Override + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { + return children.get(0).getStringValueInComplexTypeForQuery(options); } public void setNotFold(boolean notFold) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 0ea9a5dc23e912..e881874c515300 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -765,8 +765,8 @@ public String getStringValue(Type type) { } @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } public void roundCeiling(int newScale) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java index 6fbfc175ea039f..41d64c27ae6ea6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java @@ -266,7 +266,7 @@ public int compareLiteral(LiteralExpr expr) { } @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { return value.toPlainString(); } @@ -280,11 +280,6 @@ public String getStringValue() { return value.toPlainString(); } - @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); - } - @Override public long getLongValue() { return value.longValue(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index 0b5e68e1f30346..a7cfad25d465fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -2185,21 +2185,33 @@ public String getStringValue() { return ""; } - public String getStringValueInFe(FormatOptions options) { + /** + * This method is used for constant fold of query in FE, + * for different serde dialect(hive, presto, doris). + */ + public String getStringValueForQuery(FormatOptions options) { return getStringValue(); } - public String getStringValueForStreamLoad(FormatOptions options) { - return getStringValue(); + /** + * This method is to return the string value of this expr in a complex type for query + * It is only used for "getStringValueForQuery()" + * For most of the integer types, it is same as getStringValueForQuery(). + * But for others like StringLiteral and DateLiteral, it should be wrapped with quotations. + * eg: 1,2,abc,[1,2,3],["abc","def"],{10:20},{"abc":20} + */ + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { + return getStringValueForQuery(options); } - // A special method only for array literal, all primitive type in array - // will be wrapped by double quote. eg: - // ["1", "2", "3"] - // ["a", "b", "c"] - // [["1", "2", "3"], ["1"], ["3"]] - public String getStringValueForArray(FormatOptions options) { - return null; + /** + * This method is to return the string value of this expr for stream load. + * so there is a little different from "getStringValueForQuery()". + * eg, for NullLiteral, it should be "\N" for stream load, but "null" for FE constant + * for StructLiteral, the value should not contain sub column's name. + */ + public String getStringValueForStreamLoad(FormatOptions options) { + return getStringValueForQuery(options); } public final TExpr normalize(Normalizer normalizer) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java index 645afe52f99491..074fc250967820 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java @@ -155,7 +155,7 @@ public String getStringValue() { } @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { if (type == Type.TIME || type == Type.TIMEV2) { // FloatLiteral used to represent TIME type, here we need to remove apostrophe from timeStr // for example '11:22:33' -> 11:22:33 @@ -170,13 +170,12 @@ public String getStringValueInFe(FormatOptions options) { } @Override - public String getStringValueForArray(FormatOptions options) { - String ret = getStringValue(); + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { + String ret = this.getStringValueForQuery(options); if (type == Type.TIME || type == Type.TIMEV2) { - // here already wrapped in '' - ret = ret.substring(1, ret.length() - 1); + ret = options.getNestedStringWrapper() + ret + options.getNestedStringWrapper(); } - return options.getNestedStringWrapper() + ret + options.getNestedStringWrapper(); + return ret; } public static Type getDefaultTimeType(Type type) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java index 92063e6b9b30c3..33a273ad52f664 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java @@ -140,7 +140,7 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java index bb986e0ffe7a2c..f33f1103d95e2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java @@ -105,7 +105,7 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java index 978c864434dce4..a33a1b741ec8a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java @@ -20,7 +20,6 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.FormatOptions; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.util.ByteBufferUtil; import org.apache.doris.qe.ConnectContext; @@ -284,11 +283,6 @@ public String getStringValue() { return Long.toString(value); } - @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); - } - @Override public long getLongValue() { return value; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java index e568fafe45aa5e..050ec16a3304e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java @@ -101,7 +101,7 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java index 861aeaa043efcf..f499d58e046823 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java @@ -20,7 +20,6 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.FormatOptions; import org.apache.doris.common.io.Text; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; @@ -197,11 +196,6 @@ public String getStringValue() { return value.toString(); } - @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); - } - @Override public long getLongValue() { return value.longValue(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 301b6277725247..6f3ca405d95c2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -114,32 +114,6 @@ public static LiteralExpr create(String value, Type type) throws AnalysisExcepti return literalExpr; } - public static String getStringLiteralForComplexType(Expr v, FormatOptions options) { - if (!(v instanceof NullLiteral) && v.getType().isScalarType() - && (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType())) - || v.getType() == Type.BOOLEAN)) { - return v.getStringValueInFe(options); - } else if (v.getType().isComplexType()) { - // these type should also call getStringValueInFe which should handle special case for itself - return v.getStringValueInFe(options); - } else { - return v.getStringValueForArray(options); - } - } - - public static String getStringLiteralForStreamLoad(Expr v, FormatOptions options) { - if (!(v instanceof NullLiteral) && v.getType().isScalarType() - && (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType())) - || v.getType() == Type.BOOLEAN)) { - return v.getStringValueInFe(options); - } else if (v.getType().isComplexType()) { - // these type should also call getStringValueInFe which should handle special case for itself - return v.getStringValueForStreamLoad(options); - } else { - return v.getStringValueForArray(options); - } - } - /** * Init LiteralExpr's Type information * only use in rewrite alias function @@ -265,13 +239,10 @@ public int compareTo(LiteralExpr literalExpr) { @Override public abstract String getStringValue(); - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { return getStringValue(); } - @Override - public abstract String getStringValueForArray(FormatOptions options); - public long getLongValue() { return 0; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java index f330dfa6edb8bc..396883699c5f65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java @@ -171,29 +171,20 @@ private String getStringValue(Expr expr) { } @Override - public String getStringValueForArray(FormatOptions options) { - List list = new ArrayList<>(children.size()); - for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) { - list.add(children.get(i).getStringValueForArray(options) - + options.getMapKeyDelim() - + children.get(i + 1).getStringValueForArray(options)); - } - return "{" + StringUtils.join(list, ", ") + "}"; - } - - @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { List list = new ArrayList<>(children.size()); + ++options.level; for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) { // we should use type to decide we output array is suitable for json format if (children.get(i).getType().isComplexType()) { // map key type do not support complex type throw new UnsupportedOperationException("Unsupported key type for MAP: " + children.get(i).getType()); } - list.add(getStringLiteralForComplexType(children.get(i), options) - + options.getMapKeyDelim() + getStringLiteralForComplexType(children.get(i + 1), options)); + list.add(children.get(i).getStringValueInComplexTypeForQuery(options) + + options.getMapKeyDelim() + children.get(i + 1).getStringValueInComplexTypeForQuery(options)); } - return "{" + StringUtils.join(list, ", ") + "}"; + --options.level; + return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java index f3f2f24a77cd96..03c4653833d5d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java @@ -69,7 +69,7 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java index ff784b4546a609..70840bb4fdb648 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.FormatOptions; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; @@ -106,14 +107,19 @@ public String getStringValue() { } @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { return null; } + @Override + public String getStringValueForStreamLoad(FormatOptions options) { + return FeConstants.null_string; + } + // the null value inside an array is represented as "null", for exampe: // [null, null]. Not same as other primitive type to represent as \N. @Override - public String getStringValueForArray(FormatOptions options) { + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { return options.getNullFormat(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java index 0e5aeeb362b5fd..eda712552e86e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java @@ -191,8 +191,8 @@ public Expr reset() { } @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index 09cb50bb9ed72b..51f1bb53b95ebe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -154,8 +154,8 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { - return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper(); + protected String getStringValueInComplexTypeForQuery(FormatOptions options) { + return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java index 5d888168821d2f..78437c26ce07cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java @@ -107,15 +107,9 @@ public String getStringValue() { } @Override - public String getStringValueForArray(FormatOptions options) { - List list = new ArrayList<>(children.size()); - children.forEach(v -> list.add(v.getStringValueForArray(options))); - return "{" + StringUtils.join(list, ", ") + "}"; - } - - @Override - public String getStringValueInFe(FormatOptions options) { + public String getStringValueForQuery(FormatOptions options) { List list = new ArrayList<>(children.size()); + ++options.level; // same with be default field index start with 1 for (int i = 0; i < children.size(); i++) { Expr child = children.get(i); @@ -123,16 +117,21 @@ public String getStringValueInFe(FormatOptions options) { + ((StructType) type).getFields().get(i).getName() + options.getNestedStringWrapper() + options.getMapKeyDelim() - + getStringLiteralForComplexType(child, options)); + + child.getStringValueInComplexTypeForQuery(options)); } - return "{" + StringUtils.join(list, ", ") + "}"; + --options.level; + return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } @Override public String getStringValueForStreamLoad(FormatOptions options) { List list = new ArrayList<>(children.size()); - children.forEach(v -> list.add(getStringLiteralForComplexType(v, options))); - return "{" + StringUtils.join(list, ", ") + "}"; + // same with be default field index start with 1 + for (int i = 0; i < children.size(); i++) { + Expr child = children.get(i); + list.add(child.getStringValueInComplexTypeForQuery(options)); + } + return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java b/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java index a63b83ab71da98..39d939a6a8eeda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FormatOptions.java @@ -27,12 +27,24 @@ public class FormatOptions { private String nestedStringWrapper; private String mapKeyDelim; + // the string format of null value in complex type private String nullFormat; + private String collectionDelim; + // isBoolValue = true means the boolean column in collection type(array, map, ...) will print as 0 or 1. + // false means to print as true/false + // This is only for boolean column within the collection type. + // For top level boolean column, it is always 0/1 + private boolean isBoolValueNum; + // Indicate the nested level of column. It is used to control some behavior of serde + public int level = 0; - public FormatOptions(String nestedStringWrapper, String mapKeyDelim, String nullFormat) { + private FormatOptions(String nestedStringWrapper, String mapKeyDelim, String nullFormat, String collectionDelim, + boolean isBoolValueNum) { this.nestedStringWrapper = nestedStringWrapper; this.mapKeyDelim = mapKeyDelim; this.nullFormat = nullFormat; + this.collectionDelim = collectionDelim; + this.isBoolValueNum = isBoolValueNum; } public String getNestedStringWrapper() { @@ -47,11 +59,23 @@ public String getNullFormat() { return this.nullFormat; } + public String getCollectionDelim() { + return collectionDelim; + } + + public boolean isBoolValueNum() { + return isBoolValueNum; + } + public static FormatOptions getDefault() { - return new FormatOptions("\"", ":", "null"); + return new FormatOptions("\"", ":", "null", ", ", true); } public static FormatOptions getForPresto() { - return new FormatOptions("", "=", "NULL"); + return new FormatOptions("", "=", "NULL", ", ", true); + } + + public static FormatOptions getForHive() { + return new FormatOptions("\"", ":", "null", ",", false); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 5f6b74a597b8bb..0a96e577d74f1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -809,8 +809,10 @@ private void setFormatOptions() { case "trino": statementContext.setFormatOptions(FormatOptions.getForPresto()); break; - case "doris": case "hive": + statementContext.setFormatOptions(FormatOptions.getForHive()); + break; + case "doris": statementContext.setFormatOptions(FormatOptions.getDefault()); break; default: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index e4b38a311c0844..efa363d8d4015e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -53,7 +53,6 @@ import org.apache.doris.nereids.trees.expressions.DefaultValueSlot; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.Plan; @@ -176,15 +175,7 @@ public static InternalService.PDataRow getRowStringValue(List c throw new AnalysisException( "do not support non-literal expr in transactional insert operation: " + expr.toSql()); } - if (expr instanceof NullLiteral) { - row.addColBuilder().setValue(StmtExecutor.NULL_VALUE_FOR_LOAD); - } else if (expr instanceof ArrayLiteral) { - row.addColBuilder().setValue(String.format("\"%s\"", - ((ArrayLiteral) expr).toLegacyLiteral().getStringValueForArray(options))); - } else { - row.addColBuilder().setValue(String.format("\"%s\"", - ((Literal) expr).toLegacyLiteral().getStringValue())); - } + row.addColBuilder().setValue(((Literal) expr).toLegacyLiteral().getStringValueForStreamLoad(options)); } return row.build(); } @@ -253,7 +244,6 @@ private static void beginBatchInsertTransaction(ConnectContext ctx, .setTimeout((int) timeoutSecond) .setTimezone(timeZone) .setSendBatchParallelism(sendBatchParallelism) - .setTrimDoubleQuotes(true) .setSequenceCol(columns.stream() .filter(c -> Column.SEQUENCE_COL.equalsIgnoreCase(c.getName())) .map(Column::getName) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java index edde1c17c7a8ad..a053b509b13aa6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOneRowRelation.java @@ -155,7 +155,8 @@ public Optional computeResultInFe( if (expr instanceof Literal) { LiteralExpr legacyExpr = ((Literal) expr).toLegacyLiteral(); columns.add(new Column(output.getName(), output.getDataType().toCatalogDataType())); - data.add(legacyExpr.getStringValueInFe(cascadesContext.getStatementContext().getFormatOptions())); + data.add(legacyExpr.getStringValueForQuery( + cascadesContext.getStatementContext().getFormatOptions())); } else { return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index feb8c45787a013..e79a934f0468d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -616,7 +616,7 @@ public Optional handleQueryInFe(StatementBase parsedStmt) { String columnName = columnLabels.get(i); if (expr instanceof LiteralExpr) { columns.add(new Column(columnName, expr.getType())); - data.add(((LiteralExpr) expr).getStringValueInFe(options)); + data.add(((LiteralExpr) expr).getStringValueForQuery(options)); } else { return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index f1868176c2c488..4cd9fc50d13215 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -24,7 +24,6 @@ import org.apache.doris.analysis.AnalyzeStmt; import org.apache.doris.analysis.AnalyzeTblStmt; import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.ArrayLiteral; import org.apache.doris.analysis.CreateRoutineLoadStmt; import org.apache.doris.analysis.CreateTableAsSelectStmt; import org.apache.doris.analysis.CreateTableLikeStmt; @@ -43,7 +42,6 @@ import org.apache.doris.analysis.LoadType; import org.apache.doris.analysis.LockTablesStmt; import org.apache.doris.analysis.NativeInsertStmt; -import org.apache.doris.analysis.NullLiteral; import org.apache.doris.analysis.OutFileClause; import org.apache.doris.analysis.PartitionNames; import org.apache.doris.analysis.PlaceHolderExpr; @@ -270,7 +268,6 @@ public class StmtExecutor { private static final AtomicLong STMT_ID_GENERATOR = new AtomicLong(0); public static final int MAX_DATA_TO_SEND_FOR_TXN = 100; - public static final String NULL_VALUE_FOR_LOAD = "\\N"; private static Set blockSqlAstNames = Sets.newHashSet(); private Pattern beIpPattern = Pattern.compile("\\[(\\d+):"); @@ -371,19 +368,7 @@ public static InternalService.PDataRow getRowStringValue(List cols, throw new UserException( "do not support non-literal expr in transactional insert operation: " + expr.toSql()); } - if (expr instanceof NullLiteral) { - row.addColBuilder().setValue(NULL_VALUE_FOR_LOAD); - } else if (expr instanceof ArrayLiteral) { - row.addColBuilder().setValue("\"" + expr.getStringValueForStreamLoad(options) + "\""); - } else { - String stringValue = expr.getStringValueForStreamLoad(options); - if (stringValue.equals(NULL_VALUE_FOR_LOAD) || stringValue.startsWith("\"") || stringValue.endsWith( - "\"")) { - row.addColBuilder().setValue("\"" + stringValue + "\""); - } else { - row.addColBuilder().setValue(stringValue); - } - } + row.addColBuilder().setValue(expr.getStringValueForStreamLoad(options)); } return row.build(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java index d1d7a3c736e779..2085a407388cce 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java @@ -26,10 +26,13 @@ public class ArrayLiteralTest { @Test - public void testGetStringValueForArray() throws AnalysisException { + public void testGetStringForQuery() throws AnalysisException { FormatOptions options = FormatOptions.getDefault(); IntLiteral intLiteral1 = new IntLiteral(1); FloatLiteral floatLiteral = new FloatLiteral("2.15"); + FloatLiteral floatLiteral1 = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), + FloatLiteral.getDefaultTimeType(Type.TIME)); + BoolLiteral boolLiteral = new BoolLiteral(true); StringLiteral stringLiteral = new StringLiteral("shortstring"); LargeIntLiteral largeIntLiteral = new LargeIntLiteral("1000000000000000000000"); @@ -37,49 +40,71 @@ public void testGetStringValueForArray() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[\"1\", \"2.15\"]", arrayLiteral1.getStringValueForArray(options)); + Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueForQuery(options)); + ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); + Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueForQuery(options)); ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); - Assert.assertEquals("[\"1\", \"1\"]", arrayLiteral2.getStringValueForArray(options)); + Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueForQuery(options)); ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueForArray(options)); + Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueForQuery(options)); ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); - Assert.assertEquals("[\"1000000000000000000000\", \"1000000000000000000000\"]", arrayLiteral4.getStringValueForArray(options)); + Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", + arrayLiteral4.getStringValueForQuery(options)); ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueForArray(options)); + Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueForQuery(options)); ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueForArray(options)); + Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueForQuery(options)); ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); - Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]", arrayLiteral7.getStringValueForArray(options)); + Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]", + arrayLiteral7.getStringValueForQuery(options)); ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); Assert.assertEquals("[[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"], [\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]]", - arrayLiteral8.getStringValueForArray(options)); + arrayLiteral8.getStringValueForQuery(options)); ArrayLiteral arrayLiteral9 = new ArrayLiteral(); - Assert.assertEquals("[]", arrayLiteral9.getStringValueForArray(options)); + Assert.assertEquals("[]", arrayLiteral9.getStringValueForQuery(options)); + + DecimalLiteral decimalLiteral = new DecimalLiteral("1.0"); + DecimalLiteral decimalLiteral2 = new DecimalLiteral("2"); + ArrayLiteral arrayLiteral10 = new ArrayLiteral(decimalLiteral, decimalLiteral2); + Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueForQuery(options)); + + //array(1, null) + IntLiteral intLiteralWithNull = new IntLiteral(1); + ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); + Assert.assertEquals("[1, null]", arrayLiteral11.getStringValueForQuery(options)); + //array(null, 1) + ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); + Assert.assertEquals("[null, 1]", arrayLiteral12.getStringValueForQuery(options)); ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); - ArrayLiteral arrayLiteral10 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[\"1\", \"2.15\"], [\"1\", \"2.15\"]]", arrayLiteral10.getStringValueForArray(options)); - ArrayLiteral arrayLiteral11 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{\"1\":\"2.15\"}]", arrayLiteral11.getStringValueForArray(options)); - ArrayLiteral arrayLiteral12 = new ArrayLiteral(structLiteral); - Assert.assertEquals("[{\"1\", \"2.15\", \"2022-10-10\"}]", arrayLiteral12.getStringValueForArray(options)); + ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); + Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", + arrayLiteral13.getStringValueForQuery(options)); + ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); + Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueForQuery(options)); + ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); + Assert.assertEquals("[{\"col1\":1, \"col2\":2.15, \"col3\":\"2022-10-10\"}]", + arrayLiteral15.getStringValueForQuery(options)); } @Test - public void testGetStringValueForArrayForPresto() throws AnalysisException { + public void testGetStringForQueryForPresto() throws AnalysisException { FormatOptions options = FormatOptions.getForPresto(); IntLiteral intLiteral1 = new IntLiteral(1); FloatLiteral floatLiteral = new FloatLiteral("2.15"); + FloatLiteral floatLiteral1 = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), + FloatLiteral.getDefaultTimeType(Type.TIME)); + BoolLiteral boolLiteral = new BoolLiteral(true); StringLiteral stringLiteral = new StringLiteral("shortstring"); LargeIntLiteral largeIntLiteral = new LargeIntLiteral("1000000000000000000000"); @@ -87,47 +112,64 @@ public void testGetStringValueForArrayForPresto() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1, 2.15]", arrayLiteral1.getStringValueForArray(options)); + Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueForQuery(options)); + ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); + Assert.assertEquals("[11:22:33]", arrayLiteralWithTime.getStringValueForQuery(options)); ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); - Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueForArray(options)); + Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueForQuery(options)); ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[shortstring, shortstring]", arrayLiteral3.getStringValueForArray(options)); + Assert.assertEquals("[shortstring, shortstring]", arrayLiteral3.getStringValueForQuery(options)); ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); - Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", arrayLiteral4.getStringValueForArray(options)); + Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", + arrayLiteral4.getStringValueForQuery(options)); ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[NULL, NULL]", arrayLiteral5.getStringValueForArray(options)); + Assert.assertEquals("[NULL, NULL]", arrayLiteral5.getStringValueForQuery(options)); ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[2022-10-10, 2022-10-10]", arrayLiteral6.getStringValueForArray(options)); + Assert.assertEquals("[2022-10-10, 2022-10-10]", arrayLiteral6.getStringValueForQuery(options)); ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); - Assert.assertEquals("[2022-10-10 12:10:10, 2022-10-10 12:10:10]", arrayLiteral7.getStringValueForArray(options)); + Assert.assertEquals("[2022-10-10 12:10:10, 2022-10-10 12:10:10]", + arrayLiteral7.getStringValueForQuery(options)); ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); Assert.assertEquals("[[2022-10-10 12:10:10, 2022-10-10 12:10:10], [2022-10-10 12:10:10, 2022-10-10 12:10:10]]", - arrayLiteral8.getStringValueForArray(options)); + arrayLiteral8.getStringValueForQuery(options)); ArrayLiteral arrayLiteral9 = new ArrayLiteral(); - Assert.assertEquals("[]", arrayLiteral9.getStringValueForArray(options)); + Assert.assertEquals("[]", arrayLiteral9.getStringValueForQuery(options)); + + DecimalLiteral decimalLiteral = new DecimalLiteral("1.0"); + DecimalLiteral decimalLiteral2 = new DecimalLiteral("2"); + ArrayLiteral arrayLiteral10 = new ArrayLiteral(decimalLiteral, decimalLiteral2); + Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueForQuery(options)); + + //array(1, null) + IntLiteral intLiteralWithNull = new IntLiteral(1); + ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); + Assert.assertEquals("[1, NULL]", arrayLiteral11.getStringValueForQuery(options)); + //array(null, 1) + ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); + Assert.assertEquals("[NULL, 1]", arrayLiteral12.getStringValueForQuery(options)); ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); - ArrayLiteral arrayLiteral10 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[1, 2.15], [1, 2.15]]", arrayLiteral10.getStringValueForArray(options)); - ArrayLiteral arrayLiteral11 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{1=2.15}]", arrayLiteral11.getStringValueForArray(options)); - ArrayLiteral arrayLiteral12 = new ArrayLiteral(structLiteral); - Assert.assertEquals("[{1, 2.15, 2022-10-10}]", arrayLiteral12.getStringValueForArray(options)); + ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); + Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", arrayLiteral13.getStringValueForQuery(options)); + ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); + Assert.assertEquals("[{1=2.15}]", arrayLiteral14.getStringValueForQuery(options)); + ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); + Assert.assertEquals("[{col1=1, col2=2.15, col3=2022-10-10}]", arrayLiteral15.getStringValueForQuery(options)); } @Test - public void testGetStringInFe() throws AnalysisException { - FormatOptions options = FormatOptions.getDefault(); + public void testGetStringForQueryForHive() throws AnalysisException { + FormatOptions options = FormatOptions.getForHive(); IntLiteral intLiteral1 = new IntLiteral(1); FloatLiteral floatLiteral = new FloatLiteral("2.15"); FloatLiteral floatLiteral1 = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), @@ -140,64 +182,66 @@ public void testGetStringInFe() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueInFe(options)); + Assert.assertEquals("[1.0,2.15]", arrayLiteral1.getStringValueForQuery(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); - Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueInFe(options)); + Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueForQuery(options)); ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); - Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueInFe(options)); + Assert.assertEquals("[true,true]", arrayLiteral2.getStringValueForQuery(options)); ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueInFe(options)); + Assert.assertEquals("[\"shortstring\",\"shortstring\"]", arrayLiteral3.getStringValueForQuery(options)); ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); - Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", - arrayLiteral4.getStringValueInFe(options)); + Assert.assertEquals("[1000000000000000000000,1000000000000000000000]", + arrayLiteral4.getStringValueForQuery(options)); ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueInFe(options)); + Assert.assertEquals("[null,null]", arrayLiteral5.getStringValueForQuery(options)); ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueInFe(options)); + Assert.assertEquals("[\"2022-10-10\",\"2022-10-10\"]", arrayLiteral6.getStringValueForQuery(options)); ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); - Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]", - arrayLiteral7.getStringValueInFe(options)); + Assert.assertEquals("[\"2022-10-10 12:10:10\",\"2022-10-10 12:10:10\"]", + arrayLiteral7.getStringValueForQuery(options)); ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); - Assert.assertEquals("[[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"], [\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]]", - arrayLiteral8.getStringValueInFe(options)); + Assert.assertEquals( + "[[\"2022-10-10 12:10:10\",\"2022-10-10 12:10:10\"],[\"2022-10-10 12:10:10\",\"2022-10-10 12:10:10\"]]", + arrayLiteral8.getStringValueForQuery(options)); ArrayLiteral arrayLiteral9 = new ArrayLiteral(); - Assert.assertEquals("[]", arrayLiteral9.getStringValueInFe(options)); + Assert.assertEquals("[]", arrayLiteral9.getStringValueForQuery(options)); DecimalLiteral decimalLiteral = new DecimalLiteral("1.0"); DecimalLiteral decimalLiteral2 = new DecimalLiteral("2"); ArrayLiteral arrayLiteral10 = new ArrayLiteral(decimalLiteral, decimalLiteral2); - Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueInFe(options)); + Assert.assertEquals("[1.0,2.0]", arrayLiteral10.getStringValueForQuery(options)); //array(1, null) IntLiteral intLiteralWithNull = new IntLiteral(1); ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); - Assert.assertEquals("[1, null]", arrayLiteral11.getStringValueInFe(options)); + Assert.assertEquals("[1,null]", arrayLiteral11.getStringValueForQuery(options)); //array(null, 1) ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); - Assert.assertEquals("[null, 1]", arrayLiteral12.getStringValueInFe(options)); + Assert.assertEquals("[null,1]", arrayLiteral12.getStringValueForQuery(options)); ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[\"1\", \"2.15\"], [\"1\", \"2.15\"]]", arrayLiteral13.getStringValueForArray(options)); + Assert.assertEquals("[[1.0,2.15],[1.0,2.15]]", arrayLiteral13.getStringValueForQuery(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{\"1\":\"2.15\"}]", arrayLiteral14.getStringValueForArray(options)); + Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueForQuery(options)); ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); - Assert.assertEquals("[{\"1\", \"2.15\", \"2022-10-10\"}]", arrayLiteral15.getStringValueForArray(options)); + Assert.assertEquals("[{\"col1\":1,\"col2\":2.15,\"col3\":\"2022-10-10\"}]", + arrayLiteral15.getStringValueForQuery(options)); } @Test - public void testGetStringInFeForPresto() throws AnalysisException { - FormatOptions options = FormatOptions.getForPresto(); + public void testGetStringForStreamLoad() throws AnalysisException { + FormatOptions options = FormatOptions.getDefault(); IntLiteral intLiteral1 = new IntLiteral(1); FloatLiteral floatLiteral = new FloatLiteral("2.15"); FloatLiteral floatLiteral1 = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), @@ -210,58 +254,61 @@ public void testGetStringInFeForPresto() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueInFe(options)); + Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); - Assert.assertEquals("[11:22:33]", arrayLiteralWithTime.getStringValueInFe(options)); + Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral, boolLiteral); - Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueInFe(options)); + Assert.assertEquals("[1, 1]", arrayLiteral2.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral3 = new ArrayLiteral(stringLiteral, stringLiteral); - Assert.assertEquals("[shortstring, shortstring]", arrayLiteral3.getStringValueInFe(options)); + Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral); Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", - arrayLiteral4.getStringValueInFe(options)); + arrayLiteral4.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral); - Assert.assertEquals("[NULL, NULL]", arrayLiteral5.getStringValueInFe(options)); + Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral6 = new ArrayLiteral(dateLiteral, dateLiteral); - Assert.assertEquals("[2022-10-10, 2022-10-10]", arrayLiteral6.getStringValueInFe(options)); + Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral); - Assert.assertEquals("[2022-10-10 12:10:10, 2022-10-10 12:10:10]", - arrayLiteral7.getStringValueInFe(options)); + Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]", + arrayLiteral7.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7); - Assert.assertEquals("[[2022-10-10 12:10:10, 2022-10-10 12:10:10], [2022-10-10 12:10:10, 2022-10-10 12:10:10]]", - arrayLiteral8.getStringValueInFe(options)); + Assert.assertEquals( + "[[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"], [\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]]", + arrayLiteral8.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral9 = new ArrayLiteral(); - Assert.assertEquals("[]", arrayLiteral9.getStringValueInFe(options)); + Assert.assertEquals("[]", arrayLiteral9.getStringValueForStreamLoad(options)); DecimalLiteral decimalLiteral = new DecimalLiteral("1.0"); DecimalLiteral decimalLiteral2 = new DecimalLiteral("2"); ArrayLiteral arrayLiteral10 = new ArrayLiteral(decimalLiteral, decimalLiteral2); - Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueInFe(options)); + Assert.assertEquals("[1.0, 2.0]", arrayLiteral10.getStringValueForStreamLoad(options)); //array(1, null) IntLiteral intLiteralWithNull = new IntLiteral(1); ArrayLiteral arrayLiteral11 = new ArrayLiteral(intLiteralWithNull, nullLiteral); - Assert.assertEquals("[1, NULL]", arrayLiteral11.getStringValueInFe(options)); + Assert.assertEquals("[1, null]", arrayLiteral11.getStringValueForStreamLoad(options)); //array(null, 1) ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull); - Assert.assertEquals("[NULL, 1]", arrayLiteral12.getStringValueInFe(options)); + Assert.assertEquals("[null, 1]", arrayLiteral12.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[1, 2.15], [1, 2.15]]", arrayLiteral13.getStringValueForArray(options)); + Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", + arrayLiteral13.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); - Assert.assertEquals("[{1=2.15}]", arrayLiteral14.getStringValueForArray(options)); + Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); - Assert.assertEquals("[{1, 2.15, 2022-10-10}]", arrayLiteral15.getStringValueForArray(options)); + Assert.assertEquals("[{\"col1\":1, \"col2\":2.15, \"col3\":\"2022-10-10\"}]", + arrayLiteral15.getStringValueForStreamLoad(options)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java index 57357952bc68f6..1fb0f78e420a55 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java @@ -35,9 +35,9 @@ public class DateLiteralTest { @Test public void testGetStringInFe() throws AnalysisException { DateLiteral literal = new DateLiteral("1997-10-07", Type.DATE); - String s = literal.getStringValueInFe(FormatOptions.getDefault()); + String s = literal.getStringValueForQuery(FormatOptions.getDefault()); Assert.assertEquals(s, "1997-10-07"); - Assert.assertEquals(literal.getStringValueInFe(FormatOptions.getForPresto()), "1997-10-07"); + Assert.assertEquals(literal.getStringValueForQuery(FormatOptions.getForPresto()), "1997-10-07"); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/DecimalLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/DecimalLiteralTest.java index dd6213c330f6bb..4dfcb789fc5700 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/DecimalLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/DecimalLiteralTest.java @@ -36,9 +36,10 @@ public class DecimalLiteralTest { public void testGetStringInFe() { BigDecimal decimal = new BigDecimal("-123456789123456789.123456789"); DecimalLiteral literal = new DecimalLiteral(decimal); - String s = literal.getStringValueInFe(FormatOptions.getDefault()); + String s = literal.getStringValueForQuery(FormatOptions.getDefault()); Assert.assertEquals("-123456789123456789.123456789", s); - Assert.assertEquals("-123456789123456789.123456789", literal.getStringValueInFe(FormatOptions.getForPresto())); + Assert.assertEquals("-123456789123456789.123456789", + literal.getStringValueForQuery(FormatOptions.getForPresto())); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/FloatLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/FloatLiteralTest.java index 63fbea30bf7ee8..6dd4113e3898e8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/FloatLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/FloatLiteralTest.java @@ -30,13 +30,13 @@ public class FloatLiteralTest { public void testGetStringInFe() throws AnalysisException { FloatLiteral literal = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), FloatLiteral.getDefaultTimeType(Type.TIME)); - String s = literal.getStringValueInFe(FormatOptions.getDefault()); + String s = literal.getStringValueForQuery(FormatOptions.getDefault()); Assert.assertEquals("11:22:33", s); - Assert.assertEquals("11:22:33", literal.getStringValueInFe(FormatOptions.getForPresto())); + Assert.assertEquals("11:22:33", literal.getStringValueForQuery(FormatOptions.getForPresto())); FloatLiteral literal1 = new FloatLiteral(11.22); - String s1 = literal1.getStringValueInFe(FormatOptions.getDefault()); + String s1 = literal1.getStringValueForQuery(FormatOptions.getDefault()); Assert.assertEquals("11.22", s1); - Assert.assertEquals("11.22", literal1.getStringValueInFe(FormatOptions.getForPresto())); + Assert.assertEquals("11.22", literal1.getStringValueForQuery(FormatOptions.getForPresto())); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java index c36d9fcc0d2583..9e483e5177117d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java @@ -58,34 +58,36 @@ public static void setUp() throws AnalysisException { arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); mapLiteral = new MapLiteral(intLiteral1, floatLiteral); structLiteral = new StructLiteral(intLiteral1, floatLiteral, decimalLiteral1, dateLiteral); - } @Test - public void testGetStringValueForArray() throws AnalysisException { + public void testGetStringForQuery() throws AnalysisException { FormatOptions options = FormatOptions.getDefault(); MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{\"1\":\"2.15\"}", mapLiteral1.getStringValueForArray(options)); + Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueForQuery(options)); + MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); + Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueForQuery(options)); MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{\"1\":\"shortstring\"}", mapLiteral2.getStringValueForArray(options)); + Assertions.assertEquals("{1:\"shortstring\"}", mapLiteral2.getStringValueForQuery(options)); MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{\"1000000000000000000000\":\"2022-10-10\"}", mapLiteral3.getStringValueForArray(options)); - MapLiteral mapLiteral4 = new MapLiteral(nullLiteral, nullLiteral); - Assertions.assertEquals("{null:null}", mapLiteral4.getStringValueForArray(options)); + Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", mapLiteral3.getStringValueForQuery(options)); + MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); + Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueForQuery(options)); MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", - mapLiteral5.getStringValueForArray(options)); - - MapLiteral mapLiteral6 = new MapLiteral(); - Assertions.assertEquals("{}", mapLiteral6.getStringValueForArray(options)); + mapLiteral5.getStringValueForQuery(options)); + MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); + Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueForQuery(options)); - MapLiteral mapLiteral7 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{null:\"1\"}", mapLiteral7.getStringValueForArray(options)); - MapLiteral mapLiteral8 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{\"1\":null}", mapLiteral8.getStringValueForArray(options)); + MapLiteral mapLiteral7 = new MapLiteral(); + Assertions.assertEquals("{}", mapLiteral7.getStringValueForQuery(options)); + MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); + Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueForQuery(options)); + MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); + Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueForQuery(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{\"1\":[\"1\", \"2.15\"]}", mapLiteral10.getStringValueForArray(options)); + Assertions.assertEquals("{1:[1.0, 2.15]}", mapLiteral10.getStringValueForQuery(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -93,8 +95,8 @@ public void testGetStringValueForArray() throws AnalysisException { + "detailMessage = Invalid key type in Map, not support array", e.getMessage()); } - MapLiteral mapLiteral11 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{\"1.0\":{\"1\":\"2.15\"}}", mapLiteral11.getStringValueForArray(options)); + MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); + Assertions.assertEquals("{1.0:{1:2.15}}", mapLiteral12.getStringValueForQuery(options)); try { new MapLiteral(mapLiteral, decimalLiteral1); } catch (Exception e) { @@ -103,40 +105,44 @@ public void testGetStringValueForArray() throws AnalysisException { } MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); - Assertions.assertEquals("{\"shortstring\":{\"1\", \"2.15\", \"1.0\", \"2022-10-10\"}}", - mapLiteral13.getStringValueForArray(options)); + Assertions.assertEquals("{\"shortstring\":{\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", + mapLiteral13.getStringValueForQuery(options)); try { new MapLiteral(structLiteral, stringLiteral); } catch (Exception e) { - Assertions.assertEquals("errCode = 2, detailMessage = Invalid key type in Map, " + Assertions.assertEquals("errCode = 2, " + + "detailMessage = Invalid key type in Map, " + "not support struct", e.getMessage()); } } @Test - public void testGetStringValueForArrayForPresto() throws AnalysisException { + public void testGetStringForQueryForPresto() throws AnalysisException { FormatOptions options = FormatOptions.getForPresto(); MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{1=2.15}", mapLiteral1.getStringValueForArray(options)); + Assertions.assertEquals("{1=2.15}", mapLiteral1.getStringValueForQuery(options)); + MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); + Assertions.assertEquals("{1=11:22:33}", mapLiteral11.getStringValueForQuery(options)); MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{1=shortstring}", mapLiteral2.getStringValueForArray(options)); + Assertions.assertEquals("{1=shortstring}", mapLiteral2.getStringValueForQuery(options)); MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{1000000000000000000000=2022-10-10}", mapLiteral3.getStringValueForArray(options)); - MapLiteral mapLiteral4 = new MapLiteral(nullLiteral, nullLiteral); - Assertions.assertEquals("{NULL=NULL}", mapLiteral4.getStringValueForArray(options)); + Assertions.assertEquals("{1000000000000000000000=2022-10-10}", mapLiteral3.getStringValueForQuery(options)); + MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); + Assertions.assertEquals("{11:22:33=NULL}", mapLiteral4.getStringValueForQuery(options)); MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); - Assertions.assertEquals("{2022-10-10 12:10:10=2022-10-10}", mapLiteral5.getStringValueForArray(options)); - - MapLiteral mapLiteral6 = new MapLiteral(); - Assertions.assertEquals("{}", mapLiteral6.getStringValueForArray(options)); + Assertions.assertEquals("{2022-10-10 12:10:10=2022-10-10}", mapLiteral5.getStringValueForQuery(options)); + MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); + Assertions.assertEquals("{1.0=2}", mapLiteral6.getStringValueForQuery(options)); - MapLiteral mapLiteral7 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{NULL=1}", mapLiteral7.getStringValueForArray(options)); - MapLiteral mapLiteral8 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{1=NULL}", mapLiteral8.getStringValueForArray(options)); + MapLiteral mapLiteral7 = new MapLiteral(); + Assertions.assertEquals("{}", mapLiteral7.getStringValueForQuery(options)); + MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); + Assertions.assertEquals("{NULL=1}", mapLiteral8.getStringValueForQuery(options)); + MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); + Assertions.assertEquals("{1=NULL}", mapLiteral9.getStringValueForQuery(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1=[1, 2.15]}", mapLiteral10.getStringValueForArray(options)); + Assertions.assertEquals("{1=[1.0, 2.15]}", mapLiteral10.getStringValueForQuery(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -144,8 +150,8 @@ public void testGetStringValueForArrayForPresto() throws AnalysisException { + "detailMessage = Invalid key type in Map, not support array", e.getMessage()); } - MapLiteral mapLiteral11 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{1.0={1=2.15}}", mapLiteral11.getStringValueForArray(options)); + MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); + Assertions.assertEquals("{1.0={1=2.15}}", mapLiteral12.getStringValueForQuery(options)); try { new MapLiteral(mapLiteral, decimalLiteral1); } catch (Exception e) { @@ -154,43 +160,45 @@ public void testGetStringValueForArrayForPresto() throws AnalysisException { } MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); - Assertions.assertEquals("{shortstring={1, 2.15, 1.0, 2022-10-10}}", - mapLiteral13.getStringValueForArray(options)); + Assertions.assertEquals("{shortstring={col1=1, col2=2.15, col3=1.0, col4=2022-10-10}}", + mapLiteral13.getStringValueForQuery(options)); try { new MapLiteral(structLiteral, stringLiteral); } catch (Exception e) { - Assertions.assertEquals("errCode = 2, detailMessage = Invalid key type in Map, " + Assertions.assertEquals("errCode = 2, " + + "detailMessage = Invalid key type in Map, " + "not support struct", e.getMessage()); } } @Test - public void testGetStringInFe() throws AnalysisException { - FormatOptions options = FormatOptions.getDefault(); + public void testGetStringForQueryForHive() throws AnalysisException { + FormatOptions options = FormatOptions.getForHive(); MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueInFe(options)); + Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueForQuery(options)); MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); - Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueInFe(options)); + Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueForQuery(options)); MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{1:\"shortstring\"}", mapLiteral2.getStringValueInFe(options)); + Assertions.assertEquals("{true:\"shortstring\"}", mapLiteral2.getStringValueForQuery(options)); MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", mapLiteral3.getStringValueInFe(options)); + Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", mapLiteral3.getStringValueForQuery(options)); MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); - Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueInFe(options)); + Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueForQuery(options)); MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); - Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", mapLiteral5.getStringValueInFe(options)); + Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", + mapLiteral5.getStringValueForQuery(options)); MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); - Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueInFe(options)); + Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueForQuery(options)); MapLiteral mapLiteral7 = new MapLiteral(); - Assertions.assertEquals("{}", mapLiteral7.getStringValueInFe(options)); + Assertions.assertEquals("{}", mapLiteral7.getStringValueForQuery(options)); MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueInFe(options)); + Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueForQuery(options)); MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueInFe(options)); + Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueForQuery(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{\"1\":[\"1\", \"2.15\"]}", mapLiteral10.getStringValueForArray(options)); + Assertions.assertEquals("{1:[1.0,2.15]}", mapLiteral10.getStringValueForQuery(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -199,7 +207,7 @@ public void testGetStringInFe() throws AnalysisException { } MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{\"1.0\":{\"1\":\"2.15\"}}", mapLiteral12.getStringValueForArray(options)); + Assertions.assertEquals("{1.0:{1:2.15}}", mapLiteral12.getStringValueForQuery(options)); try { new MapLiteral(mapLiteral, decimalLiteral1); } catch (Exception e) { @@ -208,8 +216,8 @@ public void testGetStringInFe() throws AnalysisException { } MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); - Assertions.assertEquals("{\"shortstring\":{\"1\", \"2.15\", \"1.0\", \"2022-10-10\"}}", - mapLiteral13.getStringValueForArray(options)); + Assertions.assertEquals("{\"shortstring\":{\"col1\":1,\"col2\":2.15,\"col3\":1.0,\"col4\":\"2022-10-10\"}}", + mapLiteral13.getStringValueForQuery(options)); try { new MapLiteral(structLiteral, stringLiteral); } catch (Exception e) { @@ -220,32 +228,34 @@ public void testGetStringInFe() throws AnalysisException { } @Test - public void testGetStringInFeForPresto() throws AnalysisException { - FormatOptions options = FormatOptions.getForPresto(); + public void testGetStringForStreamLoad() throws AnalysisException { + FormatOptions options = FormatOptions.getDefault(); MapLiteral mapLiteral1 = new MapLiteral(intLiteral1, floatLiteral); - Assertions.assertEquals("{1=2.15}", mapLiteral1.getStringValueInFe(options)); + Assertions.assertEquals("{1:2.15}", mapLiteral1.getStringValueForStreamLoad(options)); MapLiteral mapLiteral11 = new MapLiteral(intLiteral1, floatLiteral1); - Assertions.assertEquals("{1=11:22:33}", mapLiteral11.getStringValueInFe(options)); + Assertions.assertEquals("{1:\"11:22:33\"}", mapLiteral11.getStringValueForStreamLoad(options)); MapLiteral mapLiteral2 = new MapLiteral(boolLiteral, stringLiteral); - Assertions.assertEquals("{1=shortstring}", mapLiteral2.getStringValueInFe(options)); + Assertions.assertEquals("{1:\"shortstring\"}", mapLiteral2.getStringValueForStreamLoad(options)); MapLiteral mapLiteral3 = new MapLiteral(largeIntLiteral, dateLiteral); - Assertions.assertEquals("{1000000000000000000000=2022-10-10}", mapLiteral3.getStringValueInFe(options)); + Assertions.assertEquals("{1000000000000000000000:\"2022-10-10\"}", + mapLiteral3.getStringValueForStreamLoad(options)); MapLiteral mapLiteral4 = new MapLiteral(floatLiteral1, nullLiteral); - Assertions.assertEquals("{11:22:33=NULL}", mapLiteral4.getStringValueInFe(options)); + Assertions.assertEquals("{\"11:22:33\":null}", mapLiteral4.getStringValueForStreamLoad(options)); MapLiteral mapLiteral5 = new MapLiteral(datetimeLiteral, dateLiteral); - Assertions.assertEquals("{2022-10-10 12:10:10=2022-10-10}", mapLiteral5.getStringValueInFe(options)); + Assertions.assertEquals("{\"2022-10-10 12:10:10\":\"2022-10-10\"}", + mapLiteral5.getStringValueForStreamLoad(options)); MapLiteral mapLiteral6 = new MapLiteral(decimalLiteral1, decimalLiteral2); - Assertions.assertEquals("{1.0=2}", mapLiteral6.getStringValueInFe(options)); + Assertions.assertEquals("{1.0:2}", mapLiteral6.getStringValueForStreamLoad(options)); MapLiteral mapLiteral7 = new MapLiteral(); - Assertions.assertEquals("{}", mapLiteral7.getStringValueInFe(options)); + Assertions.assertEquals("{}", mapLiteral7.getStringValueForStreamLoad(options)); MapLiteral mapLiteral8 = new MapLiteral(nullLiteral, intLiteral1); - Assertions.assertEquals("{NULL=1}", mapLiteral8.getStringValueInFe(options)); + Assertions.assertEquals("{null:1}", mapLiteral8.getStringValueForStreamLoad(options)); MapLiteral mapLiteral9 = new MapLiteral(intLiteral1, nullLiteral); - Assertions.assertEquals("{1=NULL}", mapLiteral9.getStringValueInFe(options)); + Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueForStreamLoad(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1=[1, 2.15]}", mapLiteral10.getStringValueForArray(options)); + Assertions.assertEquals("{1:[1.0, 2.15]}", mapLiteral10.getStringValueForStreamLoad(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -254,7 +264,7 @@ public void testGetStringInFeForPresto() throws AnalysisException { } MapLiteral mapLiteral12 = new MapLiteral(decimalLiteral1, mapLiteral); - Assertions.assertEquals("{1.0={1=2.15}}", mapLiteral12.getStringValueForArray(options)); + Assertions.assertEquals("{1.0:{1:2.15}}", mapLiteral12.getStringValueForStreamLoad(options)); try { new MapLiteral(mapLiteral, decimalLiteral1); } catch (Exception e) { @@ -263,8 +273,8 @@ public void testGetStringInFeForPresto() throws AnalysisException { } MapLiteral mapLiteral13 = new MapLiteral(stringLiteral, structLiteral); - Assertions.assertEquals("{shortstring={1, 2.15, 1.0, 2022-10-10}}", - mapLiteral13.getStringValueForArray(options)); + Assertions.assertEquals("{\"shortstring\":{\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", + mapLiteral13.getStringValueForStreamLoad(options)); try { new MapLiteral(structLiteral, stringLiteral); } catch (Exception e) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ScalarLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ScalarLiteralTest.java new file mode 100644 index 00000000000000..55f94050d6f62e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ScalarLiteralTest.java @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.FormatOptions; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class ScalarLiteralTest { + static IntLiteral intLiteral1; + static FloatLiteral floatLiteral; + static FloatLiteral floatLiteral1; + static BoolLiteral boolLiteral; + static StringLiteral stringLiteral; + static LargeIntLiteral largeIntLiteral; + static NullLiteral nullLiteral; + static DateLiteral dateLiteral; + static DateLiteral datetimeLiteral; + static DecimalLiteral decimalLiteral1; + static DecimalLiteral decimalLiteral2; + static ArrayLiteral arrayLiteral; + static MapLiteral mapLiteral; + static StructLiteral structLiteral; + + @BeforeAll + public static void setUp() throws AnalysisException { + intLiteral1 = new IntLiteral(1); + floatLiteral = new FloatLiteral("2.15"); + floatLiteral1 = new FloatLiteral((double) (11 * 3600 + 22 * 60 + 33), + FloatLiteral.getDefaultTimeType(Type.TIME)); + boolLiteral = new BoolLiteral(true); + stringLiteral = new StringLiteral("shortstring"); + largeIntLiteral = new LargeIntLiteral("1000000000000000000000"); + nullLiteral = new NullLiteral(); + dateLiteral = new DateLiteral("2022-10-10", Type.DATE); + datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); + decimalLiteral1 = new DecimalLiteral("1.0"); + decimalLiteral2 = new DecimalLiteral("2"); + arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral); + mapLiteral = new MapLiteral(intLiteral1, floatLiteral); + structLiteral = new StructLiteral(intLiteral1, floatLiteral, decimalLiteral1, dateLiteral); + } + + @Test + public void testGetStringForQuery() { + FormatOptions options = FormatOptions.getDefault(); + Assertions.assertEquals("1", intLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2.15", floatLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("11:22:33", floatLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("1", boolLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("shortstring", stringLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1000000000000000000000", largeIntLiteral.getStringValueForQuery(options)); + Assertions.assertEquals(null, nullLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10", dateLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10 12:10:10", datetimeLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1.0", decimalLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2", decimalLiteral2.getStringValueForQuery(options)); + } + + @Test + public void testGetStringForQueryForPresto() { + FormatOptions options = FormatOptions.getForPresto(); + Assertions.assertEquals("1", intLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2.15", floatLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("11:22:33", floatLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("1", boolLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("shortstring", stringLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1000000000000000000000", largeIntLiteral.getStringValueForQuery(options)); + Assertions.assertEquals(null, nullLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10", dateLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10 12:10:10", datetimeLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1.0", decimalLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2", decimalLiteral2.getStringValueForQuery(options)); + } + + @Test + public void testGetStringForQueryForHive() { + FormatOptions options = FormatOptions.getForHive(); + Assertions.assertEquals("1", intLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2.15", floatLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("11:22:33", floatLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("1", boolLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("shortstring", stringLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1000000000000000000000", largeIntLiteral.getStringValueForQuery(options)); + Assertions.assertEquals(null, nullLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10", dateLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("2022-10-10 12:10:10", datetimeLiteral.getStringValueForQuery(options)); + Assertions.assertEquals("1.0", decimalLiteral1.getStringValueForQuery(options)); + Assertions.assertEquals("2", decimalLiteral2.getStringValueForQuery(options)); + } + + @Test + public void testGetStringForStreamLoad() { + FormatOptions options = FormatOptions.getDefault(); + Assertions.assertEquals("1", intLiteral1.getStringValueForStreamLoad(options)); + Assertions.assertEquals("2.15", floatLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("11:22:33", floatLiteral1.getStringValueForStreamLoad(options)); + Assertions.assertEquals("1", boolLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("shortstring", stringLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("1000000000000000000000", largeIntLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals(FeConstants.null_string, nullLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("2022-10-10", dateLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("2022-10-10 12:10:10", datetimeLiteral.getStringValueForStreamLoad(options)); + Assertions.assertEquals("1.0", decimalLiteral1.getStringValueForStreamLoad(options)); + Assertions.assertEquals("2", decimalLiteral2.getStringValueForStreamLoad(options)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java index 7e1702d2f7f6b0..5df4dc38e88021 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java @@ -61,80 +61,90 @@ public static void setUp() throws AnalysisException { } @Test - public void testGetStringValueForArray() throws AnalysisException { + public void testGetStringInFe() throws AnalysisException { FormatOptions options = FormatOptions.getDefault(); StructLiteral structLiteral1 = new StructLiteral(intLiteral1, floatLiteral, floatLiteral1, boolLiteral, - stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, - datetimeLiteral); - Assert.assertEquals("{\"1\", \"2.15\", \"11:22:33\", \"1\", \"shortstring\", " - + "\"1000000000000000000000\", \"1.0\", \"2\", \"2022-10-10\", \"2022-10-10 12:10:10\"}", - structLiteral1.getStringValueForArray(options)); + stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, datetimeLiteral); + Assert.assertEquals("{\"col1\":1, \"col2\":2.15, \"col3\":\"11:22:33\", \"col4\":1, \"col5\":" + + "\"shortstring\", \"col6\":1000000000000000000000, \"col7\":1.0, \"col8\":2, \"col9\":\"2022-10-10\", \"col10\":\"2022-10-10 12:10:10\"}", + structLiteral1.getStringValueForQuery(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); - Assert.assertEquals("{[\"1\", \"2.15\"], {\"1\":\"2.15\"}, {\"1\", \"2.15\", \"1.0\", \"2022-10-10\"}}", - structLiteral2.getStringValueForArray(options)); + Assert.assertEquals("{\"col1\":[1.0, 2.15], \"col2\":{1:2.15}, \"col3\":" + + "{\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", + structLiteral2.getStringValueForQuery(options)); StructLiteral structLiteral3 = new StructLiteral(); - Assert.assertEquals("{}", structLiteral3.getStringValueForArray(options)); + Assert.assertEquals("{}", structLiteral3.getStringValueForQuery(options)); StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); - Assert.assertEquals("{null, \"1\"}", nullStruct.getStringValueForArray(options)); + Assert.assertEquals("{\"col1\":null, \"col2\":1}", nullStruct.getStringValueForQuery(options)); + StructLiteral boolStruct = new StructLiteral(boolLiteral, intLiteral1); + Assert.assertEquals("{\"col1\":1, \"col2\":1}", boolStruct.getStringValueForQuery(options)); } @Test - public void testGetStringInFe() throws AnalysisException { - FormatOptions options = FormatOptions.getDefault(); + public void testGetStringInFeForPresto() throws AnalysisException { + FormatOptions options = FormatOptions.getForPresto(); StructLiteral structLiteral1 = new StructLiteral(intLiteral1, floatLiteral, floatLiteral1, boolLiteral, stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, datetimeLiteral); - Assert.assertEquals("{\"col1\":1, \"col2\":2.15, \"col3\":\"11:22:33\", \"col4\":1, \"col5\":" - + "\"shortstring\", \"col6\":1000000000000000000000, \"col7\":1.0, \"col8\":2, \"col9\":\"2022-10-10\", \"col10\":\"2022-10-10 12:10:10\"}", - structLiteral1.getStringValueInFe(options)); + Assert.assertEquals("{col1=1, col2=2.15, col3=11:22:33, col4=1, col5=" + + "shortstring, col6=1000000000000000000000, col7=1.0, col8=2, col9=2022-10-10, col10=2022-10-10 12:10:10}", + structLiteral1.getStringValueForQuery(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); - Assert.assertEquals("{\"col1\":[1.0, 2.15], \"col2\":{1:2.15}, \"col3\":" - + "{\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", - structLiteral2.getStringValueInFe(options)); + Assert.assertEquals("{col1=[1.0, 2.15], col2={1=2.15}, col3=" + + "{col1=1, col2=2.15, col3=1.0, col4=2022-10-10}}", + structLiteral2.getStringValueForQuery(options)); StructLiteral structLiteral3 = new StructLiteral(); - Assert.assertEquals("{}", structLiteral3.getStringValueInFe(options)); + Assert.assertEquals("{}", structLiteral3.getStringValueForQuery(options)); StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); - Assert.assertEquals("{\"col1\":null, \"col2\":1}", nullStruct.getStringValueInFe(options)); + Assert.assertEquals("{col1=NULL, col2=1}", nullStruct.getStringValueForQuery(options)); + + StructLiteral boolStruct = new StructLiteral(boolLiteral, intLiteral1); + Assert.assertEquals("{col1=1, col2=1}", boolStruct.getStringValueForQuery(options)); } @Test - public void testGetStringValueForArrayForPreto() throws AnalysisException { - FormatOptions options = FormatOptions.getForPresto(); + public void testGetStringInFeForHive() throws AnalysisException { + FormatOptions options = FormatOptions.getForHive(); StructLiteral structLiteral1 = new StructLiteral(intLiteral1, floatLiteral, floatLiteral1, boolLiteral, - stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, - datetimeLiteral); - Assert.assertEquals("{1, 2.15, 11:22:33, 1, shortstring, " - + "1000000000000000000000, 1.0, 2, 2022-10-10, 2022-10-10 12:10:10}", - structLiteral1.getStringValueForArray(options)); + stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, datetimeLiteral); + Assert.assertEquals( + "{\"col1\":1,\"col2\":2.15,\"col3\":\"11:22:33\",\"col4\":true,\"col5\":\"shortstring\",\"col6\":1000000000000000000000,\"col7\":1.0,\"col8\":2,\"col9\":\"2022-10-10\",\"col10\":\"2022-10-10 12:10:10\"}", + structLiteral1.getStringValueForQuery(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); - Assert.assertEquals("{[1, 2.15], {1=2.15}, {1, 2.15, 1.0, 2022-10-10}}", - structLiteral2.getStringValueForArray(options)); + Assert.assertEquals( + "{\"col1\":[1.0,2.15],\"col2\":{1:2.15},\"col3\":{\"col1\":1,\"col2\":2.15,\"col3\":1.0,\"col4\":\"2022-10-10\"}}", + structLiteral2.getStringValueForQuery(options)); StructLiteral structLiteral3 = new StructLiteral(); - Assert.assertEquals("{}", structLiteral3.getStringValueForArray(options)); + Assert.assertEquals("{}", structLiteral3.getStringValueForQuery(options)); StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); - Assert.assertEquals("{NULL, 1}", nullStruct.getStringValueForArray(options)); + Assert.assertEquals("{\"col1\":null,\"col2\":1}", nullStruct.getStringValueForQuery(options)); + StructLiteral boolStruct = new StructLiteral(boolLiteral, intLiteral1); + Assert.assertEquals("{\"col1\":true,\"col2\":1}", boolStruct.getStringValueForQuery(options)); } @Test - public void testGetStringInFeForPresto() throws AnalysisException { - FormatOptions options = FormatOptions.getForPresto(); + public void testGetStringForStreamLoad() throws AnalysisException { + FormatOptions options = FormatOptions.getDefault(); StructLiteral structLiteral1 = new StructLiteral(intLiteral1, floatLiteral, floatLiteral1, boolLiteral, stringLiteral, largeIntLiteral, decimalLiteral1, decimalLiteral2, dateLiteral, datetimeLiteral); - Assert.assertEquals("{col1=1, col2=2.15, col3=11:22:33, col4=1, col5=" - + "shortstring, col6=1000000000000000000000, col7=1.0, col8=2, col9=2022-10-10, col10=2022-10-10 12:10:10}", - structLiteral1.getStringValueInFe(options)); + Assert.assertEquals( + "{1, 2.15, \"11:22:33\", 1, \"shortstring\", 1000000000000000000000, 1.0, 2, \"2022-10-10\", \"2022-10-10 12:10:10\"}", + structLiteral1.getStringValueForStreamLoad(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); - Assert.assertEquals("{col1=[1.0, 2.15], col2={1=2.15}, col3=" - + "{col1=1, col2=2.15, col3=1.0, col4=2022-10-10}}", - structLiteral2.getStringValueInFe(options)); + Assert.assertEquals( + "{[1.0, 2.15], {1:2.15}, {\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", + structLiteral2.getStringValueForStreamLoad(options)); StructLiteral structLiteral3 = new StructLiteral(); - Assert.assertEquals("{}", structLiteral3.getStringValueInFe(options)); + Assert.assertEquals("{}", structLiteral3.getStringValueForStreamLoad(options)); StructLiteral nullStruct = new StructLiteral(nullLiteral, intLiteral1); - Assert.assertEquals("{col1=NULL, col2=1}", nullStruct.getStringValueInFe(options)); + Assert.assertEquals("{null, 1}", nullStruct.getStringValueForStreamLoad(options)); + + StructLiteral boolStruct = new StructLiteral(boolLiteral, intLiteral1); + Assert.assertEquals("{1, 1}", boolStruct.getStringValueForStreamLoad(options)); } } diff --git a/regression-test/data/datatype_p0/serde/test_serde_dialect_hive.out b/regression-test/data/datatype_p0/serde/test_serde_dialect_hive.out index 3ea1043cdf6f9c..aebc2e62a3b815 100644 --- a/regression-test/data/datatype_p0/serde/test_serde_dialect_hive.out +++ b/regression-test/data/datatype_p0/serde/test_serde_dialect_hive.out @@ -2,6 +2,12 @@ -- !sql01 -- 1 2 3 4 5 1.1 2.0 123456.123456789 2024-06-30 2024-06-30T10:10:11 2024-06-30T10:10:11.123456 59.50.185.152 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff this is a string with , and " abc ef 123ndedwdw true [1,2,3,4,5] [1, 2, 3, null, 5] [1.1, 2.1, 3.1, null, 5] [1.10000, 2.10000, 3.00000, null, 5.12345] ["abc", "de, f"", null, ""] [{"k1":"v1", "k2":null, "k3":"", "k4":"a , "a"}, {"k1":"v1", "k2":null, "k3 , "abc":"", "k4":"a , "a"}] [["abc", "de, f"", null, ""], [], null] \N \N {"k1":"v1", "k2":null, "k3":"", "k4":"a , "a"} {"k1":[["abc", "de, f"", null, ""], [], null], "k2":null} {10:{"k1":[["abc", "de, f"", null, ""], [], null]}, 11:null} \N {"s_id":100, "s_name":"abc , "", "s_address":null} {"s_id":null, "s_name":["abc", "de, f"", null, ""], "s_address":""} ["2024-06-01", null, "2024-06-03"] ["2024-06-01 10:10:10.000", null, "2024-06-03 01:11:23.123"] [1, 1, 0, 0, 1, 0, 0] {"s_id":100, "s_name":"abc , "", "s_gender":1} {"k1":0, "k2":1, "k3":1, "k4":0} --- !sql01 -- +-- !sql_fe01 -- +1 2 3 4 5 1.1 2.0000 123456.123456789 2024-06-30 2024-06-30 10:10:11 2024-06-30 10:10:11.123456 59.50.185.152 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff this is a string with , and " abc ef 123ndedwdw true [1, 2, 3, 4, 5] [1, 2, 3, null, 5] [1.10, 2.10, 3.10, null, 5.00] [1.10000, 2.10000, 3.00000, null, 5.12345] ["abc", "de, f"", null, ""] [{"k1":"v1", "k2":null, "k3":"", "k4":"a , "a"}, {"k1":"v1", "k2":null, "k3 , "abc":"", "k4":"a , "a"}] [["abc", "de, f"", null, ""], [], null] {"k1":"v1", "k2":null, "k3":"", "k4":"a , "a"} {"k1":[["abc", "de, f"", null, ""], [], null], "k2":null} {10:{"k1":[["abc", "de, f"", null, ""], [], null]}, 11:null} {"s_id":100, "s_name":"abc , "", "s_address":null} {"s_id":null, "s_name":["abc", "de, f"", null, ""], "s_address":""} ["2024-06-01", null, "2024-06-03"] ["2024-06-01 10:10:10", null, "2024-06-03 01:11:23.123"] [1, 1, 0, 0, 1, 0, 0] {"s_id":100, "s_name":"abc , "", "s_gender":1} {"k1":0, "k2":1, "k3":1, "k4":0} + +-- !sql02 -- 1 2 3 4 5 1.1 2.0 123456.123456789 2024-06-30 2024-06-30T10:10:11 2024-06-30T10:10:11.123456 59.50.185.152 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff this is a string with , and " abc ef 123ndedwdw true [1,2,3,4,5] [1,2,3,null,5] [1.1,2.1,3.1,null,5] [1.10000,2.10000,3.00000,null,5.12345] ["abc","de, f"",null,""] [{"k1":"v1","k2":null,"k3":"","k4":"a , "a"},{"k1":"v1","k2":null,"k3 , "abc":"","k4":"a , "a"}] [["abc","de, f"",null,""],[],null] \N \N {"k1":"v1","k2":null,"k3":"","k4":"a , "a"} {"k1":[["abc","de, f"",null,""],[],null],"k2":null} {10:{"k1":[["abc","de, f"",null,""],[],null]},11:null} \N {"s_id":100,"s_name":"abc , "","s_address":null} {"s_id":null,"s_name":["abc","de, f"",null,""],"s_address":""} ["2024-06-01",null,"2024-06-03"] ["2024-06-01 10:10:10.000",null,"2024-06-03 01:11:23.123"] [true,true,false,false,true,false,false] {"s_id":100,"s_name":"abc , "","s_gender":true} {"k1":false,"k2":true,"k3":true,"k4":false} +-- !sql_fe02 -- +1 2 3 4 5 1.1 2.0000 123456.123456789 2024-06-30 2024-06-30 10:10:11 2024-06-30 10:10:11.123456 59.50.185.152 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff this is a string with , and " abc ef 123ndedwdw true [1, 2, 3, 4, 5] [1,2,3,null,5] [1.10,2.10,3.10,null,5.00] [1.10000,2.10000,3.00000,null,5.12345] ["abc","de, f"",null,""] [{"k1":"v1","k2":null,"k3":"","k4":"a , "a"},{"k1":"v1","k2":null,"k3 , "abc":"","k4":"a , "a"}] [["abc","de, f"",null,""],[],null] {"k1":"v1","k2":null,"k3":"","k4":"a , "a"} {"k1":[["abc","de, f"",null,""],[],null],"k2":null} {10:{"k1":[["abc","de, f"",null,""],[],null]},11:null} {"s_id":100,"s_name":"abc , "","s_address":null} {"s_id":null,"s_name":["abc","de, f"",null,""],"s_address":""} ["2024-06-01",null,"2024-06-03"] ["2024-06-01 10:10:10",null,"2024-06-03 01:11:23.123"] [true,true,false,false,true,false,false] {"s_id":100,"s_name":"abc , "","s_gender":true} {"k1":false,"k2":true,"k3":true,"k4":false} + diff --git a/regression-test/suites/datatype_p0/serde/test_serde_dialect_hive.groovy b/regression-test/suites/datatype_p0/serde/test_serde_dialect_hive.groovy index b8e3037d770f7d..5e9724a8b0ac1a 100644 --- a/regression-test/suites/datatype_p0/serde/test_serde_dialect_hive.groovy +++ b/regression-test/suites/datatype_p0/serde/test_serde_dialect_hive.groovy @@ -95,10 +95,41 @@ suite("test_serde_dialect_hive", "p0") { ); """ + String constant_sql=""" + select 1,2,3,4,5,1.1,2.0000,123456.123456789,"2024-06-30", "2024-06-30 10:10:11", "2024-06-30 10:10:11.123456", + '59.50.185.152', + 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', + 'this is a string with , and "', + 'abc ef', + ' 123ndedwdw', + true, + '[1, 2, 3, 4, 5]', + [1,2,3,null,5], + [1.1,2.1,3.1,null,5.00], + [1.1,2.1,3.00000,null,5.12345], + ['abc', 'de, f"', null, ''], + [{'k1': 'v1', 'k2': null, 'k3':'', 'k4':'a , "a'}, {'k1': 'v1', 'k2': null, 'k3 , "abc':'', 'k4':'a , "a'}], + [['abc', 'de, f"', null, ''],[],null], + {'k1': 'v1', 'k2': null, 'k3':'', 'k4':'a , "a'}, + {'k1': [['abc', 'de, f"', null, ''],[],null], 'k2': null}, + {10: {'k1': [['abc', 'de, f"', null, ''],[],null]}, 11: null}, + named_struct('s_id', 100, 's_name', 'abc , "', 's_address', null), + named_struct('s_id', null, 's_name', ['abc', 'de, f"', null, ''], 's_address', ''), + ['2024-06-01',null,'2024-06-03'], + ['2024-06-01 10:10:10',null,'2024-06-03 01:11:23.123'], + [true, true, false, false, true, false, false], + named_struct('s_id', 100, 's_name', 'abc , "', 's_gender', true), + {'k1': false, 'k2': true, 'k3':true, 'k4': false} + """ + sql """set serde_dialect="doris";""" qt_sql01 """select * from test_serde_dialect_hive_tbl""" + // test fold in FE + qt_sql_fe01 """${constant_sql}""" sql """set serde_dialect="hive";""" - qt_sql01 """select * from test_serde_dialect_hive_tbl""" + qt_sql02 """select * from test_serde_dialect_hive_tbl""" + // test fold in FE + qt_sql_fe02 """${constant_sql}""" test { sql """set serde_dialect="invalid""""