Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions be/src/vec/data_types/serde/data_type_string_serde.h
Original file line number Diff line number Diff line change
Expand Up @@ -169,16 +169,16 @@ class DataTypeStringSerDeBase : public DataTypeSerDe {
Status deserialize_one_cell_from_json(IColumn& column, Slice& slice,
const FormatOptions& options) const override {
/*
* For strings in the json complex type, we remove double quotes by default.
*
* Because when querying complex types, such as selecting complexColumn from table,
* we will add double quotes to the strings in the complex type.
*
* For the map<string,int> column, insert { "abc" : 1, "hello",2 }.
* If you do not remove the double quotes, it will display {""abc"":1,""hello"": 2 },
* remove the double quotes to display { "abc" : 1, "hello",2 }.
*
*/
* For strings in the json complex type, we remove double quotes by default.
*
* Because when querying complex types, such as selecting complexColumn from table,
* we will add double quotes to the strings in the complex type.
*
* For the map<string,int> column, insert { "abc" : 1, "hello",2 }.
* If you do not remove the double quotes, it will display {""abc"":1,""hello"": 2 },
* remove the double quotes to display { "abc" : 1, "hello",2 }.
*
*/
if (_nesting_level >= 2) {
slice.trim_quote();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,42 +133,14 @@ public String getStringValue() {
}

@Override
public String getStringValueForArray(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> list.add(v.getStringValueForArray(options)));
return "[" + StringUtils.join(list, ", ") + "]";
}

@Override
public String getStringValueInFe(FormatOptions options) {
public String getStringValueForQuery(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
++options.level;
children.forEach(v -> {
String stringLiteral;
if (v instanceof NullLiteral) {
stringLiteral = options.getNullFormat();
} else {
stringLiteral = getStringLiteralForComplexType(v, options);
}
// we should use type to decide we output array is suitable for json format
list.add(stringLiteral);
list.add(v.getStringValueInComplexTypeForQuery(options));
});
return "[" + StringUtils.join(list, ", ") + "]";
}

@Override
public String getStringValueForStreamLoad(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> {
String stringLiteral;
if (v instanceof NullLiteral) {
stringLiteral = "null";
} else {
stringLiteral = getStringLiteralForStreamLoad(v, options);
}
// we should use type to decide we output array is suitable for json format
list.add(stringLiteral);
});
return "[" + StringUtils.join(list, ", ") + "]";
--options.level;
return "[" + StringUtils.join(list, options.getCollectionDelim()) + "]";
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,14 @@ public String getStringValue() {
return value ? "1" : "0";
}


@Override
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
public String getStringValueForQuery(FormatOptions options) {
if (options.level > 0) {
return options.isBoolValueNum() ? getStringValue() : (value ? "true" : "false");
} else {
return getStringValue();
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -577,8 +577,13 @@ public boolean isNullable() {
}

@Override
public String getStringValueForArray(FormatOptions options) {
return children.get(0).getStringValueForArray(options);
public String getStringValueForStreamLoad(FormatOptions options) {
return children.get(0).getStringValueForStreamLoad(options);
}

@Override
protected String getStringValueInComplexTypeForQuery(FormatOptions options) {
return children.get(0).getStringValueInComplexTypeForQuery(options);
}

public void setNotFold(boolean notFold) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -765,8 +765,8 @@ public String getStringValue(Type type) {
}

@Override
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
protected String getStringValueInComplexTypeForQuery(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper();
}

public void roundCeiling(int newScale) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ public int compareLiteral(LiteralExpr expr) {
}

@Override
public String getStringValueInFe(FormatOptions options) {
public String getStringValueForQuery(FormatOptions options) {
return value.toPlainString();
}

Expand All @@ -280,11 +280,6 @@ public String getStringValue() {
return value.toPlainString();
}

@Override
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}

@Override
public long getLongValue() {
return value.longValue();
Expand Down
32 changes: 22 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
Original file line number Diff line number Diff line change
Expand Up @@ -2185,21 +2185,33 @@ public String getStringValue() {
return "";
}

public String getStringValueInFe(FormatOptions options) {
/**
* This method is used for constant fold of query in FE,
* for different serde dialect(hive, presto, doris).
*/
public String getStringValueForQuery(FormatOptions options) {
return getStringValue();
}

public String getStringValueForStreamLoad(FormatOptions options) {
return getStringValue();
/**
* This method is to return the string value of this expr in a complex type for query
* It is only used for "getStringValueForQuery()"
* For most of the integer types, it is same as getStringValueForQuery().
* But for others like StringLiteral and DateLiteral, it should be wrapped with quotations.
* eg: 1,2,abc,[1,2,3],["abc","def"],{10:20},{"abc":20}
*/
protected String getStringValueInComplexTypeForQuery(FormatOptions options) {
return getStringValueForQuery(options);
}

// A special method only for array literal, all primitive type in array
// will be wrapped by double quote. eg:
// ["1", "2", "3"]
// ["a", "b", "c"]
// [["1", "2", "3"], ["1"], ["3"]]
public String getStringValueForArray(FormatOptions options) {
return null;
/**
* This method is to return the string value of this expr for stream load.
* so there is a little different from "getStringValueForQuery()".
* eg, for NullLiteral, it should be "\N" for stream load, but "null" for FE constant
* for StructLiteral, the value should not contain sub column's name.
*/
public String getStringValueForStreamLoad(FormatOptions options) {
return getStringValueForQuery(options);
}

public final TExpr normalize(Normalizer normalizer) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ public String getStringValue() {
}

@Override
public String getStringValueInFe(FormatOptions options) {
public String getStringValueForQuery(FormatOptions options) {
if (type == Type.TIME || type == Type.TIMEV2) {
// FloatLiteral used to represent TIME type, here we need to remove apostrophe from timeStr
// for example '11:22:33' -> 11:22:33
Expand All @@ -170,13 +170,12 @@ public String getStringValueInFe(FormatOptions options) {
}

@Override
public String getStringValueForArray(FormatOptions options) {
String ret = getStringValue();
protected String getStringValueInComplexTypeForQuery(FormatOptions options) {
String ret = this.getStringValueForQuery(options);
if (type == Type.TIME || type == Type.TIMEV2) {
// here already wrapped in ''
ret = ret.substring(1, ret.length() - 1);
ret = options.getNestedStringWrapper() + ret + options.getNestedStringWrapper();
}
return options.getNestedStringWrapper() + ret + options.getNestedStringWrapper();
return ret;
}

public static Type getDefaultTimeType(Type type) throws AnalysisException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ public String getStringValue() {
}

@Override
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
protected String getStringValueInComplexTypeForQuery(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ public String getStringValue() {
}

@Override
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
protected String getStringValueInComplexTypeForQuery(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValueForQuery(options) + options.getNestedStringWrapper();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.NotImplementedException;
import org.apache.doris.common.util.ByteBufferUtil;
import org.apache.doris.qe.ConnectContext;
Expand Down Expand Up @@ -284,11 +283,6 @@ public String getStringValue() {
return Long.toString(value);
}

@Override
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}

@Override
public long getLongValue() {
return value;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public String getStringValue() {
}

@Override
public String getStringValueForArray(FormatOptions options) {
protected String getStringValueInComplexTypeForQuery(FormatOptions options) {
return null;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.io.Text;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
Expand Down Expand Up @@ -197,11 +196,6 @@ public String getStringValue() {
return value.toString();
}

@Override
public String getStringValueForArray(FormatOptions options) {
return options.getNestedStringWrapper() + getStringValue() + options.getNestedStringWrapper();
}

@Override
public long getLongValue() {
return value.longValue();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,32 +114,6 @@ public static LiteralExpr create(String value, Type type) throws AnalysisExcepti
return literalExpr;
}

public static String getStringLiteralForComplexType(Expr v, FormatOptions options) {
if (!(v instanceof NullLiteral) && v.getType().isScalarType()
&& (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))
|| v.getType() == Type.BOOLEAN)) {
return v.getStringValueInFe(options);
} else if (v.getType().isComplexType()) {
// these type should also call getStringValueInFe which should handle special case for itself
return v.getStringValueInFe(options);
} else {
return v.getStringValueForArray(options);
}
}

public static String getStringLiteralForStreamLoad(Expr v, FormatOptions options) {
if (!(v instanceof NullLiteral) && v.getType().isScalarType()
&& (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))
|| v.getType() == Type.BOOLEAN)) {
return v.getStringValueInFe(options);
} else if (v.getType().isComplexType()) {
// these type should also call getStringValueInFe which should handle special case for itself
return v.getStringValueForStreamLoad(options);
} else {
return v.getStringValueForArray(options);
}
}

/**
* Init LiteralExpr's Type information
* only use in rewrite alias function
Expand Down Expand Up @@ -265,13 +239,10 @@ public int compareTo(LiteralExpr literalExpr) {
@Override
public abstract String getStringValue();

public String getStringValueInFe(FormatOptions options) {
public String getStringValueForQuery(FormatOptions options) {
return getStringValue();
}

@Override
public abstract String getStringValueForArray(FormatOptions options);

public long getLongValue() {
return 0;
}
Expand Down
21 changes: 6 additions & 15 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java
Original file line number Diff line number Diff line change
Expand Up @@ -171,29 +171,20 @@ private String getStringValue(Expr expr) {
}

@Override
public String getStringValueForArray(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) {
list.add(children.get(i).getStringValueForArray(options)
+ options.getMapKeyDelim()
+ children.get(i + 1).getStringValueForArray(options));
}
return "{" + StringUtils.join(list, ", ") + "}";
}

@Override
public String getStringValueInFe(FormatOptions options) {
public String getStringValueForQuery(FormatOptions options) {
List<String> list = new ArrayList<>(children.size());
++options.level;
for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) {
// we should use type to decide we output array is suitable for json format
if (children.get(i).getType().isComplexType()) {
// map key type do not support complex type
throw new UnsupportedOperationException("Unsupported key type for MAP: " + children.get(i).getType());
}
list.add(getStringLiteralForComplexType(children.get(i), options)
+ options.getMapKeyDelim() + getStringLiteralForComplexType(children.get(i + 1), options));
list.add(children.get(i).getStringValueInComplexTypeForQuery(options)
+ options.getMapKeyDelim() + children.get(i + 1).getStringValueInComplexTypeForQuery(options));
}
return "{" + StringUtils.join(list, ", ") + "}";
--options.level;
return "{" + StringUtils.join(list, options.getCollectionDelim()) + "}";
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public String getStringValue() {
}

@Override
public String getStringValueForArray(FormatOptions options) {
protected String getStringValueInComplexTypeForQuery(FormatOptions options) {
return null;
}

Expand Down
Loading
Loading