[Improve](map-type) Add contains_null for map

be/src/olap/types.h

@@ -339,7 +339,18 @@ class MapTypeInfo : public TypeInfo {
         } else if (l_size > r_size) {
             return 1;
         } else {
-            return 0;
+            // now we use collection value in array to pack map k-v
+            auto l_k = reinterpret_cast<const CollectionValue*>(l_value->key_data());
+            auto l_v = reinterpret_cast<const CollectionValue*>(l_value->value_data());
+            auto r_k = reinterpret_cast<const CollectionValue*>(r_value->key_data());
+            auto r_v = reinterpret_cast<const CollectionValue*>(r_value->value_data());
+            auto key_arr = new ArrayTypeInfo(create_static_type_info_ptr(_key_type_info.get()));
+            auto val_arr = new ArrayTypeInfo(create_static_type_info_ptr(_value_type_info.get()));
+            if (int kc = key_arr->cmp(l_k, r_k) != 0) {
+                return kc;
+            } else {
+                return val_arr->cmp(l_v, r_v);
+            }
         }
     }
 
@@ -356,7 +367,27 @@ class MapTypeInfo : public TypeInfo {
         return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
     }
 
-    std::string to_string(const void* src) const override { return "{}"; }
+    std::string to_string(const void* src) const override {
+        auto src_ = reinterpret_cast<const MapValue*>(src);
+        auto src_key = reinterpret_cast<const CollectionValue*>(src_->key_data());
+        auto src_val = reinterpret_cast<const CollectionValue*>(src_->value_data());
+        size_t key_slot_size = _key_type_info->size();
+        size_t val_slot_size = _value_type_info->size();
+        std::string result = "{";
+
+        for (size_t i = 0; i < src_key->length(); ++i) {
+            std::string k_s =
+                    _key_type_info->to_string((uint8_t*)(src_key->data()) + key_slot_size);
+            std::string v_s =
+                    _key_type_info->to_string((uint8_t*)(src_val->data()) + val_slot_size);
+            result += k_s + ":" + v_s;
+            if (i != src_key->length() - 1) {
+                result += ", ";
+            }
+        }
+        result += "}";
+        return result;
+    }
 
     void set_to_max(void* buf) const override {
         DCHECK(false) << "set_to_max of list is not implemented.";
@@ -366,7 +397,6 @@ class MapTypeInfo : public TypeInfo {
         DCHECK(false) << "set_to_min of list is not implemented.";
     }
 
-    // todo . is here only to need return 16 for two ptr?
     size_t size() const override { return sizeof(MapValue); }
 
     FieldType type() const override { return OLAP_FIELD_TYPE_MAP; }

be/src/runtime/types.cpp

@@ -55,17 +55,18 @@ TypeDescriptor::TypeDescriptor(const std::vector<TTypeNode>& types, int* idx)
     case TTypeNodeType::ARRAY: {
         DCHECK(!node.__isset.scalar_type);
         DCHECK_LT(*idx, types.size() - 1);
+        DCHECK_EQ(node.contains_nulls.size(), 1);
         type = TYPE_ARRAY;
         contains_nulls.reserve(1);
-        contains_nulls.push_back(node.contains_null);
+        contains_nulls.push_back(node.contains_nulls[0]);
         ++(*idx);
         children.push_back(TypeDescriptor(types, idx));
         break;
     }
     case TTypeNodeType::STRUCT: {
         DCHECK(!node.__isset.scalar_type);
         DCHECK_LT(*idx, types.size() - 1);
-        DCHECK(!node.__isset.contains_null);
+        DCHECK(!node.__isset.contains_nulls);
         DCHECK(node.__isset.struct_fields);
         DCHECK_GE(node.struct_fields.size(), 1);
         type = TYPE_STRUCT;
@@ -85,31 +86,19 @@ TypeDescriptor::TypeDescriptor(const std::vector<TTypeNode>& types, int* idx)
         type = TYPE_VARIANT;
         break;
     }
-    // case TTypeNodeType::STRUCT:
-    //     type = TYPE_STRUCT;
-    //     for (int i = 0; i < node.struct_fields.size(); ++i) {
-    //         ++(*idx);
-    //         children.push_back(TypeDescriptor(types, idx));
-    //         field_names.push_back(node.struct_fields[i].name);
-    //     }
-    //     break;
-    // case TTypeNodeType::ARRAY:
-    //     DCHECK(!node.__isset.scalar_type);
-    //     DCHECK_LT(*idx, types.size() - 1);
-    //     type = TYPE_ARRAY;
-    //     ++(*idx);
-    //     children.push_back(TypeDescriptor(types, idx));
-    //     break;
     case TTypeNodeType::MAP: {
         //TODO(xy): handle contains_null[0] for key and [1] for value
         DCHECK(!node.__isset.scalar_type);
         DCHECK_LT(*idx, types.size() - 2);
-        DCHECK(!node.__isset.contains_null);
+        DCHECK_EQ(node.contains_nulls.size(), 2);
+        contains_nulls.reserve(2);
         type = TYPE_MAP;
         ++(*idx);
         children.push_back(TypeDescriptor(types, idx));
+        contains_nulls.push_back(node.contains_nulls[0]);
         ++(*idx);
         children.push_back(TypeDescriptor(types, idx));
+        contains_nulls.push_back(node.contains_nulls[1]);
         break;
     }
     default:
@@ -122,11 +111,16 @@ void TypeDescriptor::to_thrift(TTypeDesc* thrift_type) const {
     TTypeNode& node = thrift_type->types.back();
     if (is_complex_type()) {
         if (type == TYPE_ARRAY) {
+            DCHECK_EQ(contains_nulls.size(), 1);
             node.type = TTypeNodeType::ARRAY;
-            node.contains_null = contains_nulls[0];
+            node.contains_nulls.reserve(1);
+            node.contains_nulls.push_back(contains_nulls[0]);
         } else if (type == TYPE_MAP) {
-            //TODO(xy): need to process children for map
+            DCHECK_EQ(contains_nulls.size(), 2);
             node.type = TTypeNodeType::MAP;
+            node.contains_nulls.reserve(2);
+            node.contains_nulls.push_back(contains_nulls[0]);
+            node.contains_nulls.push_back(contains_nulls[1]);
         } else if (type == TYPE_VARIANT) {
             node.type = TTypeNodeType::VARIANT;
         } else {

be/src/vec/data_types/data_type_factory.cpp

@@ -169,10 +169,10 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
         break;
     case TYPE_MAP:
         DCHECK(col_desc.children.size() == 2);
-        // todo. (Amory) Support Map contains_nulls in FE MapType.java Later PR
+        DCHECK_EQ(col_desc.contains_nulls.size(), 2);
         nested = std::make_shared<vectorized::DataTypeMap>(
-                create_data_type(col_desc.children[0], true),
-                create_data_type(col_desc.children[1], true));
+                create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+                create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
         break;
     case TYPE_STRUCT: {
         DCHECK(col_desc.children.size() >= 1);

be/src/vec/exprs/vmap_literal.cpp

@@ -30,17 +30,17 @@ Status VMapLiteral::prepare(RuntimeState* state, const RowDescriptor& row_desc,
     Field keys = Array();
     Field values = Array();
     // each child is slot with key1, value1, key2, value2...
-    for (int idx = 0; idx < _children.size(); ++idx) {
-        Field item;
-        ColumnPtrWrapper* const_col_wrapper = nullptr;
-        RETURN_IF_ERROR(_children[idx]->get_const_col(context, &const_col_wrapper));
-        const_col_wrapper->column_ptr->get(0, item);
+    for (int idx = 0; idx < _children.size() && idx + 1 < _children.size(); idx += 2) {
+        Field kf, vf;
+        ColumnPtrWrapper* const_key_col_wrapper = nullptr;
+        ColumnPtrWrapper* const_val_col_wrapper = nullptr;
+        RETURN_IF_ERROR(_children[idx]->get_const_col(context, &const_key_col_wrapper));
+        RETURN_IF_ERROR(_children[idx + 1]->get_const_col(context, &const_val_col_wrapper));
+        const_key_col_wrapper->column_ptr->get(0, kf);
+        const_val_col_wrapper->column_ptr->get(0, vf);
 
-        if ((idx & 1) == 0) {
-            keys.get<Array>().push_back(item);
-        } else {
-            values.get<Array>().push_back(item);
-        }
+        keys.get<Array>().push_back(kf);
+        values.get<Array>().push_back(vf);
     }
     map.get<Map>().push_back(keys);
     map.get<Map>().push_back(values);

be/src/vec/functions/function_cast.h

@@ -1542,6 +1542,7 @@ class FunctionCast final : public IFunctionBase {
         }
     }
 
+    //TODO(Amory) . Need support more cast for key , value for map
     WrapperType create_map_wrapper(const DataTypePtr& from_type, const DataTypeMap& to_type) const {
         switch (from_type->get_type_id()) {
         case TypeIndex::String:

be/test/vec/exec/parquet/parquet_thrift_test.cpp

@@ -375,13 +375,14 @@ TEST_F(ParquetThriftReaderTest, group_reader) {
         {
             TTypeNode node;
             node.__set_type(TTypeNodeType::ARRAY);
-            node.contains_null = true;
+            std::vector<bool> contains_nulls {true};
+            node.__set_contains_nulls(contains_nulls);
             TTypeNode inner;
             inner.__set_type(TTypeNodeType::SCALAR);
             TScalarType scalar_type;
             scalar_type.__set_type(TPrimitiveType::STRING);
             inner.__set_scalar_type(scalar_type);
-            inner.contains_null = true;
+            inner.__set_contains_nulls(contains_nulls);
             type.types.push_back(node);
             type.types.push_back(inner);
         }

fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java

@@ -24,6 +24,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
 import com.google.gson.annotations.SerializedName;
 
 import java.util.Objects;
@@ -138,7 +139,7 @@ public void toThrift(TTypeDesc container) {
         container.types.add(node);
         Preconditions.checkNotNull(itemType);
         node.setType(TTypeNodeType.ARRAY);
-        node.setContainsNull(containsNull);
+        node.setContainsNulls(Lists.newArrayList(containsNull));
         itemType.toThrift(container);
     }
 

fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java

@@ -24,6 +24,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
 import com.google.gson.annotations.SerializedName;
 
 import java.util.Objects;
@@ -35,19 +36,30 @@ public class MapType extends Type {
 
     @SerializedName(value = "keyType")
     private final Type keyType;
+
+    @SerializedName(value = "isKeyContainsNull")
+    private final boolean isKeyContainsNull; // Now always true
+
     @SerializedName(value = "valueType")
     private final Type valueType;
 
+    @SerializedName(value = "isValueContainsNull")
+    private final boolean isValueContainsNull; // Now always true
+
     public MapType() {
         this.keyType = NULL;
+        this.isKeyContainsNull = true;
         this.valueType = NULL;
+        this.isValueContainsNull = true;
     }
 
     public MapType(Type keyType, Type valueType) {
         Preconditions.checkNotNull(keyType);
         Preconditions.checkNotNull(valueType);
         this.keyType = keyType;
+        this.isKeyContainsNull = true;
         this.valueType = valueType;
+        this.isValueContainsNull = true;
     }
 
     @Override
@@ -59,6 +71,14 @@ public Type getKeyType() {
         return keyType;
     }
 
+    public Boolean getIsKeyContainsNull() {
+        return isKeyContainsNull;
+    }
+
+    public Boolean getIsValueContainsNull() {
+        return isValueContainsNull;
+    }
+
     public Type getValueType() {
         return valueType;
     }
@@ -141,6 +161,7 @@ public void toThrift(TTypeDesc container) {
         Preconditions.checkNotNull(keyType);
         Preconditions.checkNotNull(valueType);
         node.setType(TTypeNodeType.MAP);
+        node.setContainsNulls(Lists.newArrayList(isKeyContainsNull, isValueContainsNull));
         keyType.toThrift(container);
         valueType.toThrift(container);
     }

fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java

@@ -197,6 +197,7 @@ public abstract class Type {
         mapSubTypes.add(CHAR);
         mapSubTypes.add(VARCHAR);
         mapSubTypes.add(STRING);
+        mapSubTypes.add(NULL);
 
         structSubTypes = Lists.newArrayList();
         structSubTypes.add(BOOLEAN);
@@ -609,7 +610,7 @@ public static boolean canCastTo(Type sourceType, Type targetType) {
                 && !sourceType.isNull()) {
             // TODO: current not support cast any non-array type(except for null) to nested array type.
             return false;
-        } else if (targetType.isStructType() && sourceType.isStringType()) {
+        } else if ((targetType.isStructType() || targetType.isMapType()) && sourceType.isStringType()) {
             return true;
         } else if (sourceType.isStructType() && targetType.isStructType()) {
             return StructType.canCastTo((StructType) sourceType, (StructType) targetType);

fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java

@@ -333,7 +333,7 @@ public void analyze() throws AnalysisException {
 
         if (fn == null) {
             //TODO(xy): check map type
-            if (type.isStructType() && childType.isStringType()) {
+            if ((type.isMapType() || type.isStructType()) && childType.isStringType()) {
                 return;
             }
             if (childType.isNull() && Type.canCastTo(childType, type)) {

fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java

@@ -47,30 +47,35 @@ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
         Type keyType = Type.NULL;
         Type valueType = Type.NULL;
         children = new ArrayList<>();
-        int idx = 0;
-        // TODO(xy): limit key type to scalar type
-        for (LiteralExpr expr : exprs) {
-            if (idx % 2 == 0) {
-                if (keyType == Type.NULL) {
-                    keyType = expr.getType();
-                } else {
-                    keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), true);
-                }
-                if (keyType == Type.INVALID) {
-                    throw new AnalysisException("Invalid element type in Map");
-                }
+        // check types here
+        // 1. limit key type with map-key support
+        // 2. check type can be assigment for cast
+        for (int idx = 0; idx < exprs.length && idx + 1 < exprs.length; idx += 2) {
+            if (!MapType.MAP.supportSubType(exprs[idx].getType())) {
+                throw new AnalysisException("Invalid key type in Map, not support " + exprs[idx].getType());
+            }
+            keyType = Type.getAssignmentCompatibleType(keyType, exprs[idx].getType(), true);
+            valueType = Type.getAssignmentCompatibleType(valueType, exprs[idx + 1].getType(), true);
+        }
+
+        if (keyType == Type.INVALID) {
+            throw new AnalysisException("Invalid key type in Map.");
+        }
+        if (valueType == Type.INVALID) {
+            throw new AnalysisException("Invalid value type in Map.");
+        }
+
+        for (int idx = 0; idx < exprs.length && idx + 1 < exprs.length; idx += 2) {
+            if (exprs[idx].getType().equals(keyType)) {
+                children.add(exprs[idx]);
+            } else {
+                children.add(exprs[idx].castTo(keyType));
+            }
+            if (exprs[idx + 1].getType().equals(valueType)) {
+                children.add(exprs[idx + 1]);
             } else {
-                if (valueType == Type.NULL) {
-                    valueType = expr.getType();
-                } else {
-                    valueType = Type.getAssignmentCompatibleType(valueType, expr.getType(), true);
-                }
-                if (valueType == Type.INVALID) {
-                    throw new AnalysisException("Invalid element type in Map");
-                }
+                children.add(exprs[idx + 1].castTo(valueType));
             }
-            children.add(expr);
-            ++ idx;
         }
 
         type = new MapType(keyType, valueType);
@@ -89,13 +94,9 @@ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
         Type keyType = ((MapType) targetType).getKeyType();
         Type valueType = ((MapType) targetType).getValueType();
 
-        for (int i = 0; i < children.size(); ++ i) {
-            Expr child = children.get(i);
-            if ((i % 2) == 0) {
-                literal.children.set(i, child.uncheckedCastTo(keyType));
-            } else {
-                literal.children.set(i, child.uncheckedCastTo(valueType));
-            }
+        for (int i = 0; i < children.size() &&  i + 1 < children.size(); i += 2) {
+            literal.children.set(i, children.get(i).uncheckedCastTo(keyType));
+            literal.children.set(i + 1, children.get(i + 1).uncheckedCastTo(valueType));
         }
         literal.setType(targetType);
         return literal;
@@ -111,7 +112,7 @@ public void checkValueValid() throws AnalysisException {
     @Override
     protected String toSqlImpl() {
         List<String> list = new ArrayList<>(children.size());
-        for (int i = 0; i < children.size(); i += 2) {
+        for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) {
             list.add(children.get(i).toSqlImpl() + ":" + children.get(i + 1).toSqlImpl());
         }
         return "MAP{" + StringUtils.join(list, ", ") + "}";

fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java

@@ -217,6 +217,8 @@ public void createChildrenColumn(Type type, Column column) {
         } else if (type.isMapType()) {
             Column k = new Column(COLUMN_MAP_KEY, ((MapType) type).getKeyType());
             Column v = new Column(COLUMN_MAP_VALUE, ((MapType) type).getValueType());
+            k.setIsAllowNull(((MapType) type).getIsKeyContainsNull());
+            v.setIsAllowNull(((MapType) type).getIsValueContainsNull());
             column.addChildrenColumn(k);
             column.addChildrenColumn(v);
         } else if (type.isStructType()) {