[Branch-2.0](Serde) Fix content displayed by complex types in MySQL Client

be/src/vec/data_types/data_type.h

@@ -86,7 +86,7 @@ class IDataType : private boost::noncopyable {
     virtual Status from_string(ReadBuffer& rb, IColumn* column) const;
 
     // get specific serializer or deserializer
-    virtual DataTypeSerDeSPtr get_serde() const = 0;
+    virtual DataTypeSerDeSPtr get_serde(int nesting_level = 1) const = 0;
 
 protected:
     virtual String do_get_name() const;

be/src/vec/data_types/data_type_agg_state.h

@@ -126,7 +126,9 @@ class DataTypeAggState : public DataTypeString {
         return _agg_function->create_serialize_column();
     }
 
-    DataTypeSerDeSPtr get_serde() const override { return _agg_serialized_type->get_serde(); };
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return _agg_serialized_type->get_serde(nesting_level);
+    };
 
     DataTypePtr get_serialized_type() const { return _agg_serialized_type; }
 

be/src/vec/data_types/data_type_array.h

@@ -111,8 +111,9 @@ class DataTypeArray final : public IDataType {
     void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
     Status from_string(ReadBuffer& rb, IColumn* column) const override;
 
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeArraySerDe>(nested->get_serde());
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeArraySerDe>(nested->get_serde(nesting_level + 1),
+                                                    nesting_level);
     };
 };
 

be/src/vec/data_types/data_type_bitmap.h

@@ -113,8 +113,8 @@ class DataTypeBitMap : public IDataType {
 
     static void deserialize_as_stream(BitmapValue& value, BufferReadable& buf);
 
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeBitMapSerDe>();
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeBitMapSerDe>(nesting_level);
     };
 };
 

be/src/vec/data_types/data_type_date.h

@@ -77,7 +77,9 @@ class DataTypeDate final : public DataTypeNumberBase<Int64> {
 
     MutableColumnPtr create_column() const override;
 
-    DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeDate64SerDe>(); }
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeDate64SerDe>(nesting_level);
+    }
 };
 
 } // namespace doris::vectorized

be/src/vec/data_types/data_type_date_time.h

@@ -86,8 +86,8 @@ class DataTypeDateTime final : public DataTypeNumberBase<Int64> {
 
     std::string to_string(const IColumn& column, size_t row_num) const override;
 
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeDateTimeSerDe>();
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeDateTimeSerDe>(nesting_level);
     }
 
     Field get_field(const TExprNode& node) const override {

be/src/vec/data_types/data_type_decimal.h

@@ -254,8 +254,8 @@ class DataTypeDecimal final : public IDataType {
     std::string to_string(const IColumn& column, size_t row_num) const override;
     void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
     Status from_string(ReadBuffer& rb, IColumn* column) const override;
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeDecimalSerDe<T>>(scale, precision);
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeDecimalSerDe<T>>(scale, precision, nesting_level);
     };
 
     /// Decimal specific

be/src/vec/data_types/data_type_fixed_length_object.h

@@ -83,8 +83,8 @@ class DataTypeFixedLengthObject final : public IDataType {
     bool have_subtypes() const override { return false; }
 
     bool can_be_inside_low_cardinality() const override { return false; }
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeFixedLengthObjectSerDe>();
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeFixedLengthObjectSerDe>(nesting_level);
     };
 };
 

be/src/vec/data_types/data_type_hll.h

@@ -101,7 +101,9 @@ class DataTypeHLL : public IDataType {
 
     static void deserialize_as_stream(HyperLogLog& value, BufferReadable& buf);
 
-    DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeHLLSerDe>(); };
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeHLLSerDe>(nesting_level);
+    };
 };
 
 } // namespace doris::vectorized

be/src/vec/data_types/data_type_jsonb.h

@@ -93,7 +93,9 @@ class DataTypeJsonb final : public IDataType {
     std::string to_string(const IColumn& column, size_t row_num) const override;
     void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
     Status from_string(ReadBuffer& rb, IColumn* column) const override;
-    DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeJsonbSerDe>(); };
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeJsonbSerDe>(nesting_level);
+    };
 
 private:
     DataTypeString data_type_string;

be/src/vec/data_types/data_type_map.h

@@ -104,8 +104,10 @@ class DataTypeMap final : public IDataType {
     std::string to_string(const IColumn& column, size_t row_num) const override;
     void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
     Status from_string(ReadBuffer& rb, IColumn* column) const override;
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeMapSerDe>(key_type->get_serde(), value_type->get_serde());
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeMapSerDe>(key_type->get_serde(nesting_level + 1),
+                                                  value_type->get_serde(nesting_level + 1),
+                                                  nesting_level);
     };
 };
 

be/src/vec/data_types/data_type_nothing.h

@@ -90,7 +90,7 @@ class DataTypeNothing final : public IDataType {
     }
 
     bool have_subtypes() const override { return false; }
-    DataTypeSerDeSPtr get_serde() const override {
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
         LOG(FATAL) << get_name() << " not support serde";
     };
 };

be/src/vec/data_types/data_type_nullable.h

@@ -123,8 +123,9 @@ class DataTypeNullable final : public IDataType {
     const DataTypePtr& get_nested_type() const { return nested_data_type; }
     bool is_null_literal() const override { return nested_data_type->is_null_literal(); }
 
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeNullableSerDe>(nested_data_type->get_serde());
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeNullableSerDe>(nested_data_type->get_serde(nesting_level),
+                                                       nesting_level);
     }
 
 private:

be/src/vec/data_types/data_type_number_base.h

@@ -154,8 +154,8 @@ class DataTypeNumberBase : public IDataType {
     Status from_string(ReadBuffer& rb, IColumn* column) const override;
     bool is_null_literal() const override { return _is_null_literal; }
     void set_null_literal(bool flag) { _is_null_literal = flag; }
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeNumberSerDe<T>>();
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeNumberSerDe<T>>(nesting_level);
     };
 
 private:

be/src/vec/data_types/data_type_object.h

@@ -83,8 +83,8 @@ class DataTypeObject : public IDataType {
         LOG(FATAL) << "Unimplemented get_field for object";
     }
 
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeObjectSerDe>();
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeObjectSerDe>(nesting_level);
     };
 };
 } // namespace doris::vectorized

be/src/vec/data_types/data_type_quantilestate.h

@@ -104,8 +104,8 @@ class DataTypeQuantileState : public IDataType {
     static void serialize_as_stream(const QuantileState<T>& value, BufferWritable& buf);
 
     static void deserialize_as_stream(QuantileState<T>& value, BufferReadable& buf);
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeQuantileStateSerDe<T>>();
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeQuantileStateSerDe<T>>(nesting_level);
     };
 };
 using DataTypeQuantileStateDouble = DataTypeQuantileState<double>;

be/src/vec/data_types/data_type_string.h

@@ -91,8 +91,8 @@ class DataTypeString : public IDataType {
     std::string to_string(const IColumn& column, size_t row_num) const override;
     void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
     Status from_string(ReadBuffer& rb, IColumn* column) const override;
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeStringSerDe>();
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeStringSerDe>(nesting_level);
     };
 };
 

be/src/vec/data_types/data_type_struct.h

@@ -129,12 +129,12 @@ class DataTypeStruct final : public IDataType {
     std::string to_string(const IColumn& column, size_t row_num) const override;
     void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
     bool get_have_explicit_names() const { return have_explicit_names; }
-    DataTypeSerDeSPtr get_serde() const override {
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
         DataTypeSerDeSPtrs ptrs;
         for (auto iter = elems.begin(); iter < elems.end(); ++iter) {
-            ptrs.push_back((*iter)->get_serde());
+            ptrs.push_back((*iter)->get_serde(nesting_level + 1));
         }
-        return std::make_shared<DataTypeStructSerDe>(ptrs, names);
+        return std::make_shared<DataTypeStructSerDe>(ptrs, names, nesting_level);
     };
 };
 

be/src/vec/data_types/data_type_time.h

@@ -69,7 +69,9 @@ class DataTypeTime final : public DataTypeNumberBase<Float64> {
     bool can_be_used_in_boolean_context() const override { return true; }
     bool can_be_inside_nullable() const override { return true; }
 
-    DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeTimeSerDe>(); };
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeTimeSerDe>(nesting_level);
+    };
     TypeIndex get_type_id() const override { return TypeIndex::Time; }
     const char* get_family_name() const override { return "time"; }
 };
@@ -104,8 +106,8 @@ class DataTypeTimeV2 final : public DataTypeNumberBase<Float64> {
     bool can_be_inside_nullable() const override { return true; }
 
     void to_pb_column_meta(PColumnMeta* col_meta) const override;
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeTimeV2SerDe>(_scale);
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeTimeV2SerDe>(_scale, nesting_level);
     };
     TypeIndex get_type_id() const override { return TypeIndex::TimeV2; }
     const char* get_family_name() const override { return "timev2"; }

be/src/vec/data_types/data_type_time_v2.h

@@ -69,7 +69,9 @@ class DataTypeDateV2 final : public DataTypeNumberBase<UInt32> {
 
     bool can_be_inside_nullable() const override { return true; }
 
-    DataTypeSerDeSPtr get_serde() const override { return std::make_shared<DataTypeDateV2SerDe>(); }
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeDateV2SerDe>(nesting_level);
+    }
 
     Field get_field(const TExprNode& node) const override {
         DateV2Value<DateV2ValueType> value;
@@ -127,8 +129,8 @@ class DataTypeDateTimeV2 final : public DataTypeNumberBase<UInt64> {
     std::string to_string(const IColumn& column, size_t row_num) const override;
     void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
     Status from_string(ReadBuffer& rb, IColumn* column) const override;
-    DataTypeSerDeSPtr get_serde() const override {
-        return std::make_shared<DataTypeDateTimeV2SerDe>(_scale);
+    DataTypeSerDeSPtr get_serde(int nesting_level = 1) const override {
+        return std::make_shared<DataTypeDateTimeV2SerDe>(_scale, nesting_level);
     };
 
     Field get_field(const TExprNode& node) const override {

be/src/vec/data_types/serde/data_type_array_serde.cpp

@@ -65,15 +65,13 @@ void DataTypeArraySerDe::serialize_one_cell_to_json(const IColumn& column, int r
 Status DataTypeArraySerDe::deserialize_column_from_json_vector(IColumn& column,
                                                                std::vector<Slice>& slices,
                                                                int* num_deserialized,
-                                                               const FormatOptions& options,
-                                                               int nesting_level) const {
+                                                               const FormatOptions& options) const {
     DESERIALIZE_COLUMN_FROM_JSON_VECTOR();
     return Status::OK();
 }
 
 Status DataTypeArraySerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice,
-                                                          const FormatOptions& options,
-                                                          int nesting_level) const {
+                                                          const FormatOptions& options) const {
     if (slice.empty()) {
         return Status::InvalidArgument("slice is empty!");
     }
@@ -142,15 +140,15 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_json(IColumn& column, Slice
     }
 
     int elem_deserialized = 0;
-    Status st = nested_serde->deserialize_column_from_json_vector(
-            nested_column, slices, &elem_deserialized, options, nesting_level + 1);
+    Status st = nested_serde->deserialize_column_from_json_vector(nested_column, slices,
+                                                                  &elem_deserialized, options);
     offsets.emplace_back(offsets.back() + elem_deserialized);
     return st;
 }
 
-Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text(IColumn& column, Slice& slice,
-                                                               const FormatOptions& options,
-                                                               int nesting_level) const {
+Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text(
+        IColumn& column, Slice& slice, const FormatOptions& options,
+        int hive_text_complex_type_delimiter_level) const {
     if (slice.empty()) {
         return Status::InvalidArgument("slice is empty!");
     }
@@ -159,7 +157,8 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text(IColumn& column,
     IColumn& nested_column = array_column.get_data();
     DCHECK(nested_column.is_nullable());
 
-    char collection_delimiter = options.get_collection_delimiter(nesting_level);
+    char collection_delimiter =
+            options.get_collection_delimiter(hive_text_complex_type_delimiter_level);
 
     std::vector<Slice> slices;
     for (int idx = 0, start = 0; idx <= slice.size; idx++) {
@@ -172,23 +171,22 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text(IColumn& column,
 
     int elem_deserialized = 0;
     Status status = nested_serde->deserialize_column_from_hive_text_vector(
-            nested_column, slices, &elem_deserialized, options, nesting_level + 1);
+            nested_column, slices, &elem_deserialized, options,
+            hive_text_complex_type_delimiter_level + 1);
     offsets.emplace_back(offsets.back() + elem_deserialized);
     return status;
 }
 
-Status DataTypeArraySerDe::deserialize_column_from_hive_text_vector(IColumn& column,
-                                                                    std::vector<Slice>& slices,
-                                                                    int* num_deserialized,
-                                                                    const FormatOptions& options,
-                                                                    int nesting_level) const {
+Status DataTypeArraySerDe::deserialize_column_from_hive_text_vector(
+        IColumn& column, std::vector<Slice>& slices, int* num_deserialized,
+        const FormatOptions& options, int hive_text_complex_type_delimiter_level) const {
     DESERIALIZE_COLUMN_FROM_HIVE_TEXT_VECTOR();
     return Status::OK();
 }
 
-void DataTypeArraySerDe::serialize_one_cell_to_hive_text(const IColumn& column, int row_num,
-                                                         BufferWritable& bw, FormatOptions& options,
-                                                         int nesting_level) const {
+void DataTypeArraySerDe::serialize_one_cell_to_hive_text(
+        const IColumn& column, int row_num, BufferWritable& bw, FormatOptions& options,
+        int hive_text_complex_type_delimiter_level) const {
     auto result = check_column_const_set_readability(column, row_num);
     ColumnPtr ptr = result.first;
     row_num = result.second;
@@ -201,13 +199,13 @@ void DataTypeArraySerDe::serialize_one_cell_to_hive_text(const IColumn& column,
 
     const IColumn& nested_column = data_column.get_data();
 
-    char delimiter = options.get_collection_delimiter(nesting_level);
+    char delimiter = options.get_collection_delimiter(hive_text_complex_type_delimiter_level);
     for (size_t i = start; i < end; ++i) {
         if (i != start) {
             bw.write(delimiter);
         }
         nested_serde->serialize_one_cell_to_hive_text(nested_column, i, bw, options,
-                                                      nesting_level + 1);
+                                                      hive_text_complex_type_delimiter_level + 1);
     }
 }
 
@@ -288,7 +286,8 @@ Status DataTypeArraySerDe::_write_column_to_mysql(const IColumn& column,
             }
         }
         if (data.is_null_at(j)) {
-            if (0 != result.push_string("NULL", strlen("NULL"))) {
+            if (0 != result.push_string(NULL_IN_COMPLEX_TYPE.c_str(),
+                                        strlen(NULL_IN_COMPLEX_TYPE.c_str()))) {
                 return Status::InternalError("pack mysql buffer failed.");
             }
         } else {