From c2c76c9ab43644a55e02fbae3d20b495255c91c8 Mon Sep 17 00:00:00 2001 From: amorynan Date: Wed, 15 Feb 2023 14:43:07 +0800 Subject: [PATCH 1/8] support stream load and fix some bugs --- .../olap/rowset/segment_v2/column_writer.cpp | 13 +- be/src/olap/rowset/segment_v2/column_writer.h | 2 +- be/src/vec/data_types/data_type_factory.cpp | 4 +- be/src/vec/data_types/data_type_factory.hpp | 3 +- be/src/vec/data_types/data_type_map.cpp | 155 +++++++++++++----- be/src/vec/exprs/vexpr.cpp | 9 + be/src/vec/functions/function_cast.h | 12 ++ be/src/vec/olap/olap_data_convertor.cpp | 6 + be/src/vec/olap/olap_data_convertor.h | 3 +- be/src/vec/sink/vmysql_result_writer.cpp | 6 + 10 files changed, 159 insertions(+), 54 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp index 583becb2b0a75f..a35c840eb5bf1b 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.cpp +++ b/be/src/olap/rowset/segment_v2/column_writer.cpp @@ -1000,7 +1000,15 @@ Status MapColumnWriter::finish() { return Status::OK(); } -// todo. make keys and values write +Status MapColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t** ptr, + size_t num_rows) { + if (is_nullable()) { + RETURN_IF_ERROR(_null_writer->append_data(&null_map, num_rows)); + } + RETURN_IF_ERROR(append_data(ptr, num_rows)); + return Status::OK(); +} + Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) { auto kv_ptr = reinterpret_cast(*ptr); for (size_t i = 0; i < 2; ++i) { @@ -1008,9 +1016,6 @@ Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) { const uint8_t* val_ptr = (const uint8_t*)data; RETURN_IF_ERROR(_kv_writers[i]->append_data(&val_ptr, num_rows)); } - if (is_nullable()) { - return write_null_column(num_rows, false); - } return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h index 7d140324ddf6b5..036c4d3bafde49 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.h +++ b/be/src/olap/rowset/segment_v2/column_writer.h @@ -382,7 +382,7 @@ class MapColumnWriter final : public ColumnWriter, public FlushPageCallback { Status init() override; Status append_data(const uint8_t** ptr, size_t num_rows) override; - + Status append_nullable(const uint8_t* null_map, const uint8_t** ptr, size_t num_rows) override; uint64_t estimate_buffer_size() override; Status finish() override; diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp index 65510dd9f2dcfb..6e6c4c6b157b88 100644 --- a/be/src/vec/data_types/data_type_factory.cpp +++ b/be/src/vec/data_types/data_type_factory.cpp @@ -170,8 +170,8 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo case TYPE_MAP: DCHECK(col_desc.children.size() == 2); nested = std::make_shared( - create_data_type(col_desc.children[0], col_desc.contains_nulls[0]), - create_data_type(col_desc.children[1], col_desc.contains_nulls[1])); + create_data_type(col_desc.children[0], false), + create_data_type(col_desc.children[1], false)); break; case TYPE_STRUCT: { DCHECK(col_desc.children.size() >= 1); diff --git a/be/src/vec/data_types/data_type_factory.hpp b/be/src/vec/data_types/data_type_factory.hpp index 879418a326897c..b2623b06daf9e8 100644 --- a/be/src/vec/data_types/data_type_factory.hpp +++ b/be/src/vec/data_types/data_type_factory.hpp @@ -113,7 +113,8 @@ class DataTypeFactory { return entity.second; } } - if (type_ptr->get_type_id() == TypeIndex::Struct) { + if (type_ptr->get_type_id() == TypeIndex::Struct || + type_ptr->get_type_id() == TypeIndex::Map) { DataTypeFactory::instance().register_data_type(type_ptr->get_name(), type_ptr); for (const auto& entity : _invert_data_type_map) { if (entity.first->equals(*type_ptr)) { diff --git a/be/src/vec/data_types/data_type_map.cpp b/be/src/vec/data_types/data_type_map.cpp index c40e0362c5d1f6..6d1b838cf22315 100644 --- a/be/src/vec/data_types/data_type_map.cpp +++ b/be/src/vec/data_types/data_type_map.cpp @@ -21,13 +21,22 @@ #include "vec/columns/column_array.h" #include "vec/columns/column_map.h" #include "vec/common/assert_cast.h" -#include "vec/data_types/data_type_factory.hpp" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_nullable.h" namespace doris::vectorized { DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_) { - key_type = keys_; - value_type = values_; + if (!keys_->is_nullable()) { + key_type = make_nullable(keys_); + } else { + key_type = keys_; + } + if (!values_->is_nullable()) { + value_type = make_nullable(values_); + } else { + value_type = values_; + } keys = std::make_shared(key_type); values = std::make_shared(value_type); @@ -53,7 +62,7 @@ std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const ss << ", "; } if (nested_keys_column.is_null_at(i)) { - ss << "NULL"; + ss << "null"; } else if (WhichDataType(remove_nullable(key_type)).is_string_or_fixed_string()) { ss << "'" << key_type->to_string(nested_keys_column, i) << "'"; } else { @@ -61,7 +70,7 @@ std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const } ss << ":"; if (nested_values_column.is_null_at(i)) { - ss << "NULL"; + ss << "null"; } else if (WhichDataType(remove_nullable(value_type)).is_string_or_fixed_string()) { ss << "'" << value_type->to_string(nested_values_column, i) << "'"; } else { @@ -78,6 +87,68 @@ void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size ostr.write(ss.c_str(), strlen(ss.c_str())); } +bool next_slot_from_string(ReadBuffer& rb, StringRef& output) { + StringRef element(rb.position(), 0); + bool has_quota = false; + if (rb.eof()) { + return false; + } + + // ltrim + while (!rb.eof() && isspace(*rb.position())) { + ++rb.position(); + element.data = rb.position(); + } + + // parse string + if (*rb.position() == '"' || *rb.position() == '\'') { + const char str_sep = *rb.position(); + size_t str_len = 1; + // search until next '"' or '\'' + while (str_len < rb.count() && *(rb.position() + str_len) != str_sep) { + ++str_len; + } + // invalid string + if (str_len >= rb.count()) { + rb.position() = rb.end(); + return false; + } + has_quota = true; + rb.position() += str_len + 1; + element.size += str_len + 1; + } + + // parse array element until map separator ':' or ',' or end '}' + while (!rb.eof() && (*rb.position() != ':') && (*rb.position() != ',') && + (rb.count() != 1 || *rb.position() != '}')) { + if (has_quota && !isspace(*rb.position())) { + return false; + } + ++rb.position(); + ++element.size; + } + // invalid array element + if (rb.eof()) { + return false; + } + // adjust read buffer position to first char of next array element + ++rb.position(); + + // rtrim + while (element.size > 0 && isspace(element.data[element.size - 1])) { + --element.size; + } + + // trim '"' and '\'' for string + if (element.size >= 2 && (element.data[0] == '"' || element.data[0] == '\'') && + element.data[0] == element.data[element.size - 1]) { + ++element.data; + element.size -= 2; + } + output = element; + return true; +} + Status DataTypeMap::from_string(ReadBuffer& rb, IColumn* column) const { DCHECK(!rb.eof()); auto* map_column = assert_cast(column); @@ -91,57 +162,51 @@ Status DataTypeMap::from_string(ReadBuffer& rb, IColumn* column) const { *(rb.end() - 1)); } - std::stringstream keyCharset; - std::stringstream valCharset; - if (rb.count() == 2) { // empty map {} , need to make empty array to add offset - keyCharset << "[]"; - valCharset << "[]"; + map_column->insert_default(); } else { - // {"aaa": 1, "bbb": 20}, need to handle key and value to make key column arr and value arr + // {"aaa": 1, "bbb": 20}, need to handle key slot and value slot to make key column arr and value arr // skip "{" ++rb.position(); - keyCharset << "["; - valCharset << "["; + auto& keys_arr = reinterpret_cast(map_column->get_keys()); + ColumnArray::Offsets64& key_off = keys_arr.get_offsets(); + auto& values_arr = reinterpret_cast(map_column->get_values()); + ColumnArray::Offsets64& val_off = values_arr.get_offsets(); + + IColumn& nested_key_column = keys_arr.get_data(); + DCHECK(nested_key_column.is_nullable()); + IColumn& nested_val_column = values_arr.get_data(); + DCHECK(nested_val_column.is_nullable()); + + size_t element_num = 0; while (!rb.eof()) { - size_t kv_len = 0; - auto start = rb.position(); - while (!rb.eof() && *start != ',' && *start != '}') { - kv_len++; - start++; + StringRef key_element(rb.position(), rb.count()); + if (!next_slot_from_string(rb, key_element)) { + return Status::InvalidArgument("Cannot read map key from text '{}'", + key_element.to_string()); } - if (kv_len >= rb.count()) { - return Status::InvalidArgument("Invalid Length"); + StringRef value_element(rb.position(), rb.count()); + if (!next_slot_from_string(rb, value_element)) { + return Status::InvalidArgument("Cannot read map value from text '{}'", + value_element.to_string()); } - - size_t k_len = 0; - auto k_rb = rb.position(); - while (kv_len > 0 && *k_rb != ':') { - k_len++; - k_rb++; + ReadBuffer krb(const_cast(key_element.data), key_element.size); + ReadBuffer vrb(const_cast(value_element.data), value_element.size); + if (auto st = key_type->from_string(krb, &nested_key_column); !st.ok()) { + map_column->pop_back(element_num); + return st; + } + if (auto st = value_type->from_string(vrb, &nested_val_column); !st.ok()) { + map_column->pop_back(element_num); + return st; } - ReadBuffer key_rb(rb.position(), k_len); - ReadBuffer val_rb(k_rb + 1, kv_len - k_len - 1); - - // handle key - keyCharset << key_rb.to_string(); - keyCharset << ","; - - // handle value - valCharset << val_rb.to_string(); - valCharset << ","; - rb.position() += kv_len + 1; + ++element_num; } - keyCharset << ']'; - valCharset << ']'; + key_off.push_back(key_off.back() + element_num); + val_off.push_back(val_off.back() + element_num); } - - ReadBuffer kb(keyCharset.str().data(), keyCharset.str().length()); - ReadBuffer vb(valCharset.str().data(), valCharset.str().length()); - keys->from_string(kb, &map_column->get_keys()); - values->from_string(vb, &map_column->get_values()); return Status::OK(); } @@ -199,4 +264,4 @@ const char* DataTypeMap::deserialize(const char* buf, IColumn* column, int data_ data_version); } -} // namespace doris::vectorized +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp index 69ccf0f5bd7419..f90993884d2d6f 100644 --- a/be/src/vec/exprs/vexpr.cpp +++ b/be/src/vec/exprs/vexpr.cpp @@ -372,6 +372,15 @@ FunctionContext::TypeDesc VExpr::column_type_to_type_desc(const TypeDescriptor& out.children.push_back(VExpr::column_type_to_type_desc(t)); } break; + case TYPE_MAP: + CHECK(type.children.size() == 2); + // only support map key is scalar + CHECK(!type.children[0].is_complex_type()); + out.type = FunctionContext::TYPE_MAP; + for (const auto& t : type.children) { + out.children.push_back(VExpr::column_type_to_type_desc(t)); + } + break; case TYPE_STRING: out.type = FunctionContext::TYPE_STRING; out.len = type.len; diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index c8dc4282df7519..7ac3316fdbe9a3 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -1541,6 +1541,16 @@ class FunctionCast final : public IFunctionBase { return &ConvertImplGenericToJsonb::execute; } } + + WrapperType create_map_wrapper(const DataTypePtr& from_type, const DataTypeMap& to_type) const { + switch (from_type->get_type_id()) { + case TypeIndex::String: + return &ConvertImplGenericFromString::execute; + default: + return create_unsupport_wrapper(from_type->get_name(), to_type.get_name()); + } + } + // check struct value type and get to_type value // TODO: need handle another type to cast struct WrapperType create_struct_wrapper(const DataTypePtr& from_type, @@ -1727,6 +1737,8 @@ class FunctionCast final : public IFunctionBase { static_cast(*to_type)); case TypeIndex::Struct: return create_struct_wrapper(from_type, static_cast(*to_type)); + case TypeIndex::Map: + return create_map_wrapper(from_type, static_cast(*to_type)); default: break; } diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp index 431dda727dd337..7f27b31454acc9 100644 --- a/be/src/vec/olap/olap_data_convertor.cpp +++ b/be/src/vec/olap/olap_data_convertor.cpp @@ -786,6 +786,12 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap( return Status::OK(); } +void OlapBlockDataConvertor::OlapColumnDataConvertorMap::set_source_column( + const ColumnWithTypeAndName& typed_column, size_t row_pos, size_t num_rows) { + OlapBlockDataConvertor::OlapColumnDataConvertorBase::set_source_column(typed_column, row_pos, + num_rows); +} + Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap() { const ColumnMap* column_map = nullptr; const DataTypeMap* data_type_map = nullptr; diff --git a/be/src/vec/olap/olap_data_convertor.h b/be/src/vec/olap/olap_data_convertor.h index 0efd5c1bd8ae7f..5ec301e5fe876f 100644 --- a/be/src/vec/olap/olap_data_convertor.h +++ b/be/src/vec/olap/olap_data_convertor.h @@ -413,10 +413,11 @@ class OlapBlockDataConvertor { Status convert_to_olap() override; const void* get_data() const override { return _results.data(); }; - const void* get_data_at(size_t offset) const override { LOG(FATAL) << "now not support get_data_at for OlapColumnDataConvertorMap"; }; + void set_source_column(const ColumnWithTypeAndName& typed_column, size_t row_pos, + size_t num_rows) override; private: Status convert_to_olap(const ColumnMap* column_map, const DataTypeMap* data_type_map); diff --git a/be/src/vec/sink/vmysql_result_writer.cpp b/be/src/vec/sink/vmysql_result_writer.cpp index f74c3f94a428fc..bade8089127e37 100644 --- a/be/src/vec/sink/vmysql_result_writer.cpp +++ b/be/src/vec/sink/vmysql_result_writer.cpp @@ -195,6 +195,12 @@ Status VMysqlResultWriter::_add_one_column( return Status::InternalError("pack mysql buffer failed."); } + if constexpr (is_nullable) { + if (column_ptr->is_null_at(i)) { + buf_ret = rows_buffer[i].push_null(); + continue; + } + } rows_buffer[i].open_dynamic_mode(); std::string cell_str = map_type.to_string(*column, i); buf_ret = rows_buffer[i].push_string(cell_str.c_str(), strlen(cell_str.c_str())); From 4808e44ebd89865836a29e87c08bc37dbe2ada8c Mon Sep 17 00:00:00 2001 From: amorynan Date: Thu, 16 Feb 2023 12:52:01 +0800 Subject: [PATCH 2/8] fixed and add some regression test --- be/src/vec/data_types/data_type_factory.cpp | 5 +- be/src/vec/data_types/data_type_map.cpp | 64 ++++++---- be/src/vec/runtime/vfile_result_writer.cpp | 4 + regression-test/data/map_p0/test_map.csv | 15 +++ regression-test/data/map_p0/test_map_dml.out | 11 ++ .../map_p0/test_map_load_and_function.out | 37 ++++++ .../suites/map_p0/test_map_dml.groovy | 114 ++++++++++++++++++ .../map_p0/test_map_load_and_function.groovy | 90 ++++++++++++++ 8 files changed, 314 insertions(+), 26 deletions(-) create mode 100644 regression-test/data/map_p0/test_map.csv create mode 100644 regression-test/data/map_p0/test_map_dml.out create mode 100644 regression-test/data/map_p0/test_map_load_and_function.out create mode 100644 regression-test/suites/map_p0/test_map_dml.groovy create mode 100644 regression-test/suites/map_p0/test_map_load_and_function.groovy diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp index 6e6c4c6b157b88..91fc51187b35c8 100644 --- a/be/src/vec/data_types/data_type_factory.cpp +++ b/be/src/vec/data_types/data_type_factory.cpp @@ -169,9 +169,10 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo break; case TYPE_MAP: DCHECK(col_desc.children.size() == 2); + // todo. (Amory) Support Map contains_nulls in FE MapType.java Later PR nested = std::make_shared( - create_data_type(col_desc.children[0], false), - create_data_type(col_desc.children[1], false)); + create_data_type(col_desc.children[0], true), + create_data_type(col_desc.children[1], true)); break; case TYPE_STRUCT: { DCHECK(col_desc.children.size() >= 1); diff --git a/be/src/vec/data_types/data_type_map.cpp b/be/src/vec/data_types/data_type_map.cpp index 6d1b838cf22315..daf683c3abc11b 100644 --- a/be/src/vec/data_types/data_type_map.cpp +++ b/be/src/vec/data_types/data_type_map.cpp @@ -20,6 +20,7 @@ #include "gen_cpp/data.pb.h" #include "vec/columns/column_array.h" #include "vec/columns/column_map.h" +#include "vec/columns/column_nullable.h" #include "vec/common/assert_cast.h" #include "vec/data_types/data_type_array.h" #include "vec/data_types/data_type_nullable.h" @@ -27,16 +28,8 @@ namespace doris::vectorized { DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_) { - if (!keys_->is_nullable()) { - key_type = make_nullable(keys_); - } else { - key_type = keys_; - } - if (!values_->is_nullable()) { - value_type = make_nullable(values_); - } else { - value_type = values_; - } + key_type = make_nullable(keys_); + value_type = make_nullable(values_); keys = std::make_shared(key_type); values = std::make_shared(value_type); @@ -87,9 +80,9 @@ void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size ostr.write(ss.c_str(), strlen(ss.c_str())); } -bool next_slot_from_string(ReadBuffer& rb, StringRef& output) { +bool next_slot_from_string(ReadBuffer& rb, StringRef& output, bool& has_quota) { StringRef element(rb.position(), 0); - bool has_quota = false; + has_quota = false; if (rb.eof()) { return false; } @@ -149,6 +142,23 @@ bool next_slot_from_string(ReadBuffer& rb, StringRef& output) { return true; } +bool is_empty_null_element(StringRef element, IColumn* nested_column, bool has_quota) { + auto& nested_null_col = reinterpret_cast(*nested_column); + // handle empty element + if (element.size == 0) { + nested_null_col.get_nested_column().insert_default(); + nested_null_col.get_null_map_data().push_back(0); + return true; + } + + // handle null element + if (!has_quota && element.size == 4 && strncmp(element.data, "null", 4) == 0) { + nested_null_col.get_nested_column().insert_default(); + nested_null_col.get_null_map_data().push_back(1); + return true; + } + return false; +} Status DataTypeMap::from_string(ReadBuffer& rb, IColumn* column) const { DCHECK(!rb.eof()); auto* map_column = assert_cast(column); @@ -182,26 +192,32 @@ Status DataTypeMap::from_string(ReadBuffer& rb, IColumn* column) const { size_t element_num = 0; while (!rb.eof()) { StringRef key_element(rb.position(), rb.count()); - if (!next_slot_from_string(rb, key_element)) { + bool has_quota = false; + if (!next_slot_from_string(rb, key_element, has_quota)) { return Status::InvalidArgument("Cannot read map key from text '{}'", key_element.to_string()); } + if (!is_empty_null_element(key_element, &nested_key_column, has_quota)) { + ReadBuffer krb(const_cast(key_element.data), key_element.size); + if (auto st = key_type->from_string(krb, &nested_key_column); !st.ok()) { + map_column->pop_back(element_num); + return st; + } + } + + has_quota = false; StringRef value_element(rb.position(), rb.count()); - if (!next_slot_from_string(rb, value_element)) { + if (!next_slot_from_string(rb, value_element, has_quota)) { return Status::InvalidArgument("Cannot read map value from text '{}'", value_element.to_string()); } - ReadBuffer krb(const_cast(key_element.data), key_element.size); - ReadBuffer vrb(const_cast(value_element.data), value_element.size); - if (auto st = key_type->from_string(krb, &nested_key_column); !st.ok()) { - map_column->pop_back(element_num); - return st; - } - if (auto st = value_type->from_string(vrb, &nested_val_column); !st.ok()) { - map_column->pop_back(element_num); - return st; + if (!is_empty_null_element(value_element, &nested_val_column, has_quota)) { + ReadBuffer vrb(const_cast(value_element.data), value_element.size); + if (auto st = value_type->from_string(vrb, &nested_val_column); !st.ok()) { + map_column->pop_back(element_num); + return st; + } } - ++element_num; } key_off.push_back(key_off.back() + element_num); diff --git a/be/src/vec/runtime/vfile_result_writer.cpp b/be/src/vec/runtime/vfile_result_writer.cpp index 2782008b343cb6..16d48bde08bb2f 100644 --- a/be/src/vec/runtime/vfile_result_writer.cpp +++ b/be/src/vec/runtime/vfile_result_writer.cpp @@ -342,6 +342,10 @@ Status VFileResultWriter::_write_csv_file(const Block& block) { _plain_text_outstream << col.type->to_string(*col.column, i); break; } + case TYPE_MAP: { + _plain_text_outstream << col.type->to_string(*col.column, i); + break; + } default: { // not supported type, like BITMAP, just export null _plain_text_outstream << NULL_IN_CSV; diff --git a/regression-test/data/map_p0/test_map.csv b/regression-test/data/map_p0/test_map.csv new file mode 100644 index 00000000000000..e7ccbab130cdcc --- /dev/null +++ b/regression-test/data/map_p0/test_map.csv @@ -0,0 +1,15 @@ +1 \N +2 {" 11amory ":23, "beat":20, " clever ": 66} +3 {"k1": 31, "k2": 300} +4 {} +5 \N +6 {"k1":41, "k2": 400} +7 {" 13,amory ":2, " bet ":20, " cler ": 26} +8 {} +9 {' 1,amy ':2, " k2 ":90, " k7 ": 33} +10 {} +11 {"k1': 4, "k2": 400} +12 {"k3":23, null: 20, "k4": null} +13 {"null":1} +15 {:2, "k2":} +16 {null:null} diff --git a/regression-test/data/map_p0/test_map_dml.out b/regression-test/data/map_p0/test_map_dml.out new file mode 100644 index 00000000000000..531ce9ac2ee53c --- /dev/null +++ b/regression-test/data/map_p0/test_map_dml.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 {' amory ':6, 'happy':38} +6 {'amory':6, 'is':38, 'cl':0} + +-- !test -- +100 {1:'1', 2:'2', 3:'3'} {32767:'32767', 32768:'32768', 32769:'32769'} [65534, 65535, 65536] {2022-07-13:1} {2022-07-13 12:30:00:'2022-07-13 12:30:00'} {0.33:33, 0.67:67} + +-- !select -- +100 {1:'1', 2:'2', 3:'3'} {32767:'32767', 32768:'32768', 32769:'32769'} [65534, 65535, 65536] {2022-07-13:1} {2022-07-13 12:30:00:'2022-07-13 12:30:00'} {0.33:33, 0.67:67} + diff --git a/regression-test/data/map_p0/test_map_load_and_function.out b/regression-test/data/map_p0/test_map_load_and_function.out new file mode 100644 index 00000000000000..1f6a5383b5bb30 --- /dev/null +++ b/regression-test/data/map_p0/test_map_load_and_function.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 \N +2 {' 11amory ':23, 'beat':20, ' clever ':66} +3 {'k1':31, 'k2':300} +4 {} +5 \N +6 {'k1':41, 'k2':400} +7 {' 13,amory ':2, ' bet ':20, ' cler ':26} +8 {} +9 {' 1,amy ':2, ' k2 ':90, ' k7 ':33} +10 {} +11 \N +12 {'k3':23, null:20, 'k4':null} +13 {'null':1} +15 {'':2, 'k2':0} +16 {null:null} + +-- !select -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +300 +400 +0 +\N +\N +\N +\N +130 + diff --git a/regression-test/suites/map_p0/test_map_dml.groovy b/regression-test/suites/map_p0/test_map_dml.groovy new file mode 100644 index 00000000000000..65b30a24a52729 --- /dev/null +++ b/regression-test/suites/map_p0/test_map_dml.groovy @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_map_dml", "p0") { + // define a sql table + def testTable = "tbl_test_map_string_int" + def testTable01 = "tbl_test_map_normal" + + def create_test_table = {testTablex -> + def result1 = sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + `k1` INT(11) NULL COMMENT "", + `k2` Map NULL COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2" + ) + """ + + // DDL/DML return 1 row and 3 column, the only value is update row count + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 0, "Create table should update 0 rows") + + // insert 1 row to check whether the table is ok + def result2 = sql "INSERT INTO ${testTable} VALUES (6, {'amory': 6, 'is': 38, 'cl': 0})" + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 1) + assertTrue(result2[0][0] == 1, "Insert should update 1 rows") + } + + def create_test_table01 = {testTablez -> + def result1 = sql """ + CREATE TABLE IF NOT EXISTS ${testTable01} ( + `k1` int(11) NULL, + `k2` Map NULL, + `k3` Map NULL, + `k4` array NULL, + `k5` Map NULL, + `k6` Map NULL, + `k7` Map NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ) + """ + + // DDL/DML return 1 row and 3 column, the only value is update row count + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 0, "Create table should update 0 rows") + + + def result2 = sql """ INSERT INTO ${testTable01} VALUES (100, {1: '1', 2: '2', 3:'3'}, {32767: '32767', 32768: '32768', 32769: '32769'}, + [65534, 65535, 65536], {'2022-07-13': 1}, {'2022-07-13 12:30:00': '2022-07-13 12:30:00'}, + {0.33: 33, 0.67: 67}) + """ + + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 1) + assertTrue(result2[0][0] == 1, "Insert should update 1 rows") + + qt_test "select * from ${testTable01} order by k1" + } + + + // case1: string_int for map + try { + def res = sql "DROP TABLE IF EXISTS ${testTable}" + + create_test_table.call(testTable) + sql "INSERT INTO ${testTable} VALUES (1, {' amory ': 6, 'happy': 38})" + + // select the table and check whether the data is correct + qt_select "select * from ${testTable} order by k1" + + } finally { + try_sql("DROP TABLE IF EXISTS ${testTable}") + } + + // case2: normal key val type for map + try { + def res = sql "DROP TABLE IF EXISTS ${testTable01}" + create_test_table01.call(testTable) + // select the table and check whether the data is correct + qt_select "select * from ${testTable01} order by k1" + + } finally { + try_sql("DROP TABLE IF EXISTS ${testTable01}") + } +} diff --git a/regression-test/suites/map_p0/test_map_load_and_function.groovy b/regression-test/suites/map_p0/test_map_load_and_function.groovy new file mode 100644 index 00000000000000..b154597ca23d32 --- /dev/null +++ b/regression-test/suites/map_p0/test_map_load_and_function.groovy @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_map_load_and_function", "p0") { + // define a sql table + def testTable = "tbl_test_map" + def dataFile = "test_map.csv" + + sql "DROP TABLE IF EXISTS ${testTable}" + + sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + id INT, + m Map + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES("replication_num" = "1"); + """ + + // load the map data from csv file + streamLoad { + table testTable + + file dataFile // import csv file + time 10000 // limit inflight 10s + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals("OK", json.Message) + assertEquals(15, json.NumberTotalRows) + assertTrue(json.LoadBytes > 0) + + } + } + + // check result + qt_select "SELECT * FROM ${testTable} ORDER BY id" + + // insert into valid json rows + sql """INSERT INTO ${testTable} VALUES(12, NULL)""" + sql """INSERT INTO ${testTable} VALUES(13, {"k1":100, "k2": 130})""" + + // map element_at + qt_select "SELECT m['k2'] FROM ${testTable}" + + // map select into outfile + // check outfile + def outFilePath = """${context.file.parent}/tmp""" + logger.warn("test_map_selectOutFile the outFilePath=" + outFilePath) + + File path = new File(outFilePath) + if (path.exists()) { + for (File f: path.listFiles()) { + f.delete(); + } + path.delete(); + } + if (!path.exists()) { + assert path.mkdirs() + } + sql """ + SELECT * FROM ${testTable} INTO OUTFILE "file://${outFilePath}/"; + """ + File[] files = path.listFiles() + assert files.length == 1 +} From 0768602c7a38ad2d4a130d1e608147eab93955bd Mon Sep 17 00:00:00 2001 From: amorynan Date: Thu, 16 Feb 2023 17:54:57 +0800 Subject: [PATCH 3/8] format regression data dir --- be/src/vec/olap/olap_data_convertor.cpp | 6 - be/src/vec/olap/olap_data_convertor.h | 2 - .../data/load/insert/test_map_dml.out | 8 ++ .../stream_load}/test_map.csv | 2 +- .../test_map_load_and_function.out | 10 +- .../query_p0/show/test_map_show_create.out | 4 + .../suites/export/test_map_export.groovy | 135 ++++++++++++++++++ .../insert}/test_map_dml.groovy | 12 +- .../test_map_load_and_function.groovy | 30 +--- .../query_p0/show/test_map_show_create.groovy | 69 +++++++++ 10 files changed, 235 insertions(+), 43 deletions(-) create mode 100644 regression-test/data/load/insert/test_map_dml.out rename regression-test/data/{map_p0 => load_p0/stream_load}/test_map.csv (84%) rename regression-test/data/{map_p0 => load_p0/stream_load}/test_map_load_and_function.out (90%) create mode 100644 regression-test/data/query_p0/show/test_map_show_create.out create mode 100644 regression-test/suites/export/test_map_export.groovy rename regression-test/suites/{map_p0 => load/insert}/test_map_dml.groovy (95%) rename regression-test/suites/{map_p0 => load_p0/stream_load}/test_map_load_and_function.groovy (78%) create mode 100644 regression-test/suites/query_p0/show/test_map_show_create.groovy diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp index 7f27b31454acc9..431dda727dd337 100644 --- a/be/src/vec/olap/olap_data_convertor.cpp +++ b/be/src/vec/olap/olap_data_convertor.cpp @@ -786,12 +786,6 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap( return Status::OK(); } -void OlapBlockDataConvertor::OlapColumnDataConvertorMap::set_source_column( - const ColumnWithTypeAndName& typed_column, size_t row_pos, size_t num_rows) { - OlapBlockDataConvertor::OlapColumnDataConvertorBase::set_source_column(typed_column, row_pos, - num_rows); -} - Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap() { const ColumnMap* column_map = nullptr; const DataTypeMap* data_type_map = nullptr; diff --git a/be/src/vec/olap/olap_data_convertor.h b/be/src/vec/olap/olap_data_convertor.h index 5ec301e5fe876f..b23ba5ee5f0bf0 100644 --- a/be/src/vec/olap/olap_data_convertor.h +++ b/be/src/vec/olap/olap_data_convertor.h @@ -416,8 +416,6 @@ class OlapBlockDataConvertor { const void* get_data_at(size_t offset) const override { LOG(FATAL) << "now not support get_data_at for OlapColumnDataConvertorMap"; }; - void set_source_column(const ColumnWithTypeAndName& typed_column, size_t row_pos, - size_t num_rows) override; private: Status convert_to_olap(const ColumnMap* column_map, const DataTypeMap* data_type_map); diff --git a/regression-test/data/load/insert/test_map_dml.out b/regression-test/data/load/insert/test_map_dml.out new file mode 100644 index 00000000000000..446b0a9c5984f7 --- /dev/null +++ b/regression-test/data/load/insert/test_map_dml.out @@ -0,0 +1,8 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 {' amory ':6, 'happy':38} +6 {'amory':6, 'is':38, 'cl':0} + +-- !select -- +100 {1:'1', 2:'2', 3:'3'} {32767:'32767', 32768:'32768', 32769:'32769'} [65534, 65535, 65536] {2022-07-13:1} {2022-07-13 12:30:00:'2022-07-13 12:30:00'} {0.33:33, 0.67:67} + diff --git a/regression-test/data/map_p0/test_map.csv b/regression-test/data/load_p0/stream_load/test_map.csv similarity index 84% rename from regression-test/data/map_p0/test_map.csv rename to regression-test/data/load_p0/stream_load/test_map.csv index e7ccbab130cdcc..cbace425cc6708 100644 --- a/regression-test/data/map_p0/test_map.csv +++ b/regression-test/data/load_p0/stream_load/test_map.csv @@ -4,7 +4,7 @@ 4 {} 5 \N 6 {"k1":41, "k2": 400} -7 {" 13,amory ":2, " bet ":20, " cler ": 26} +7 {" 33,amory ":2, " bet ":20, " cler ": 26} 8 {} 9 {' 1,amy ':2, " k2 ":90, " k7 ": 33} 10 {} diff --git a/regression-test/data/map_p0/test_map_load_and_function.out b/regression-test/data/load_p0/stream_load/test_map_load_and_function.out similarity index 90% rename from regression-test/data/map_p0/test_map_load_and_function.out rename to regression-test/data/load_p0/stream_load/test_map_load_and_function.out index 1f6a5383b5bb30..1b7eb1f5a3d7cb 100644 --- a/regression-test/data/map_p0/test_map_load_and_function.out +++ b/regression-test/data/load_p0/stream_load/test_map_load_and_function.out @@ -6,7 +6,7 @@ 4 {} 5 \N 6 {'k1':41, 'k2':400} -7 {' 13,amory ':2, ' bet ':20, ' cler ':26} +7 {' 33,amory ':2, ' bet ':20, ' cler ':26} 8 {} 9 {' 1,amy ':2, ' k2 ':90, ' k7 ':33} 10 {} @@ -19,19 +19,19 @@ -- !select -- \N \N +300 \N \N +400 \N \N \N \N \N -300 -400 -0 -\N \N \N \N 130 +0 +\N diff --git a/regression-test/data/query_p0/show/test_map_show_create.out b/regression-test/data/query_p0/show/test_map_show_create.out new file mode 100644 index 00000000000000..78b706ca7d367e --- /dev/null +++ b/regression-test/data/query_p0/show/test_map_show_create.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +test_map_show_create CREATE TABLE `test_map_show_create` (\n `k1` int(11) NULL,\n `k2` MAP NULL,\n `k3` MAP NULL,\n `k4` MAP NULL,\n `k5` MAP NULL,\n `k6` MAP NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false"\n); + diff --git a/regression-test/suites/export/test_map_export.groovy b/regression-test/suites/export/test_map_export.groovy new file mode 100644 index 00000000000000..4297d068b7d3f5 --- /dev/null +++ b/regression-test/suites/export/test_map_export.groovy @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_map_export", "export") { + // check whether the FE config 'enable_outfile_to_local' is true + StringBuilder strBuilder = new StringBuilder() + strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword) + strBuilder.append(" http://" + context.config.feHttpAddress + "/rest/v1/config/fe") + + String command = strBuilder.toString() + def process = command.toString().execute() + def code = process.waitFor() + def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + def out = process.getText() + logger.info("Request FE Config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def response = parseJson(out.trim()) + assertEquals(response.code, 0) + assertEquals(response.msg, "success") + def configJson = response.data.rows + boolean enableOutfileToLocal = false + for (Object conf: configJson) { + assert conf instanceof Map + if (((Map) conf).get("Name").toLowerCase() == "enable_outfile_to_local") { + enableOutfileToLocal = ((Map) conf).get("Value").toLowerCase() == "true" + } + } + if (!enableOutfileToLocal) { + logger.warn("Please set enable_outfile_to_local to true to run test_outfile") + return + } + + // define the table + def testTable = "tbl_test_map" + + sql "DROP TABLE IF EXISTS ${testTable}" + sql "ADMIN SET FRONTEND CONFIG ('enable_map_type' = 'true')" + + sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + id INT, + m Map + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES("replication_num" = "1"); + """ + + // make data + sql """ INSERT INTO ${testTable} VALUES (1, NULL); """ + sql """ INSERT INTO ${testTable} VALUES (2, {}); """ + sql """ INSERT INTO ${testTable} VALUES (3, {" 33,amory ":2, " bet ":20, " cler ": 26}); """ + sql """ INSERT INTO ${testTable} VALUES (4, {"k3":23, null: 20, "k4": null}); """ + sql """ INSERT INTO ${testTable} VALUES (5, {:2, "k2":}); """ + + // check result + qt_select_default """ SELECT * FROM ${testTable} t ORDER BY k1; """ + + def outFilePath = """${context.file.parent}/tmp""" + logger.info("test_map_export the outFilePath=" + outFilePath) + // map select into outfile + try { + File path = new File(outFilePath) + if (path.exists()) { + for (File f : path.listFiles()) { + f.delete(); + } + path.delete(); + } + if (!path.exists()) { + assert path.mkdirs() + } + sql """ + SELECT * FROM ${testTable} ORDER BY id INTO OUTFILE "file://${outFilePath}/"; + """ + File[] files = path.listFiles() + assert files.length == 1 + + List outLines = Files.readAllLines(Paths.get(files[0].getAbsolutePath()), StandardCharsets.UTF_8) + assert outLines.size() == 5 + for (int r = 0; r < outLines.size(); r++) { + String[] outLine = outLines.get(r).split("\t") + assert outLine.size() == 2 + // check NULL + if (outLine[0] == 1) { + assert outLine[1] == "\\N" + } + // check empty + if (outLine[0] == 2) { + assert outLine[1] == "{}" + } + // check key contains ',' + if (outLine[0] == 3) { + assert outLine[1] == "{\" 33,amory \":2, \" bet \":20, \" cler \": 26}" + } + // check key val NULL + if (outLine[0] == 4) { + assert outLine[1] == "{\"k3\":23, null: 20, \"k4\": null}" + } + // check key val empty + if (outLine[0] == 5) { + assert outLine[1] == "{:2, \"k2\":}" + } + } + } finally { + try_sql("DROP TABLE IF EXISTS ${testTable}") + File path = new File(outFilePath) + if (path.exists()) { + for (File f : path.listFiles()) { + f.delete(); + } + path.delete(); + } + } +} diff --git a/regression-test/suites/map_p0/test_map_dml.groovy b/regression-test/suites/load/insert/test_map_dml.groovy similarity index 95% rename from regression-test/suites/map_p0/test_map_dml.groovy rename to regression-test/suites/load/insert/test_map_dml.groovy index 65b30a24a52729..438bd0b496d6e7 100644 --- a/regression-test/suites/map_p0/test_map_dml.groovy +++ b/regression-test/suites/load/insert/test_map_dml.groovy @@ -15,11 +15,13 @@ // specific language governing permissions and limitations // under the License. -suite("test_map_dml", "p0") { +suite("test_map_dml", "load") { // define a sql table def testTable = "tbl_test_map_string_int" def testTable01 = "tbl_test_map_normal" + sql "ADMIN SET FRONTEND CONFIG ('enable_map_type' = 'true')" + def create_test_table = {testTablex -> def result1 = sql """ CREATE TABLE IF NOT EXISTS ${testTable} ( @@ -82,20 +84,17 @@ suite("test_map_dml", "p0") { assertTrue(result2.size() == 1) assertTrue(result2[0].size() == 1) assertTrue(result2[0][0] == 1, "Insert should update 1 rows") - - qt_test "select * from ${testTable01} order by k1" } // case1: string_int for map try { def res = sql "DROP TABLE IF EXISTS ${testTable}" - create_test_table.call(testTable) sql "INSERT INTO ${testTable} VALUES (1, {' amory ': 6, 'happy': 38})" // select the table and check whether the data is correct - qt_select "select * from ${testTable} order by k1" + qt_select "SELECT * FROM ${testTable} ORDER BY k1" } finally { try_sql("DROP TABLE IF EXISTS ${testTable}") @@ -104,9 +103,10 @@ suite("test_map_dml", "p0") { // case2: normal key val type for map try { def res = sql "DROP TABLE IF EXISTS ${testTable01}" + create_test_table01.call(testTable) // select the table and check whether the data is correct - qt_select "select * from ${testTable01} order by k1" + qt_select "SELECT * FROM ${testTable01} ORDER BY k1" } finally { try_sql("DROP TABLE IF EXISTS ${testTable01}") diff --git a/regression-test/suites/map_p0/test_map_load_and_function.groovy b/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy similarity index 78% rename from regression-test/suites/map_p0/test_map_load_and_function.groovy rename to regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy index b154597ca23d32..d796c08b2eaa8e 100644 --- a/regression-test/suites/map_p0/test_map_load_and_function.groovy +++ b/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy @@ -17,12 +17,17 @@ import org.codehaus.groovy.runtime.IOGroovyMethods +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + suite("test_map_load_and_function", "p0") { // define a sql table def testTable = "tbl_test_map" def dataFile = "test_map.csv" sql "DROP TABLE IF EXISTS ${testTable}" + sql "ADMIN SET FRONTEND CONFIG ('enable_map_type' = 'true')" sql """ CREATE TABLE IF NOT EXISTS ${testTable} ( @@ -37,7 +42,7 @@ suite("test_map_load_and_function", "p0") { // load the map data from csv file streamLoad { table testTable - + file dataFile // import csv file time 10000 // limit inflight 10s @@ -65,26 +70,5 @@ suite("test_map_load_and_function", "p0") { sql """INSERT INTO ${testTable} VALUES(13, {"k1":100, "k2": 130})""" // map element_at - qt_select "SELECT m['k2'] FROM ${testTable}" - - // map select into outfile - // check outfile - def outFilePath = """${context.file.parent}/tmp""" - logger.warn("test_map_selectOutFile the outFilePath=" + outFilePath) - - File path = new File(outFilePath) - if (path.exists()) { - for (File f: path.listFiles()) { - f.delete(); - } - path.delete(); - } - if (!path.exists()) { - assert path.mkdirs() - } - sql """ - SELECT * FROM ${testTable} INTO OUTFILE "file://${outFilePath}/"; - """ - File[] files = path.listFiles() - assert files.length == 1 + qt_select "SELECT m['k2'] FROM ${testTable} ORDER BY id" } diff --git a/regression-test/suites/query_p0/show/test_map_show_create.groovy b/regression-test/suites/query_p0/show/test_map_show_create.groovy new file mode 100644 index 00000000000000..463cb2babc74d1 --- /dev/null +++ b/regression-test/suites/query_p0/show/test_map_show_create.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_map_show_create", "query") { + // define a sql table + def testTable = "test_map_show_create" + + def create_test_table = {testTablex -> + def result1 = sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + `k1` INT(11) NULL, + `k2` MAP NULL, + `k3` MAP NULL, + `k4` MAP NULL, + `k5` MAP NULL, + `k6` Map NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ) + """ + + // DDL/DML return 1 row and 3 column, the only value is update row count + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 0, "Create table should update 0 rows") + + // insert 1 row to check whether the table is ok + def result2 = sql """ INSERT INTO ${testTable} VALUES (100, {1: '1', 2: '2', 3:'3'}, + {32767: '32767', 32768: '32768', 32769: '32769'}, {'2022-07-13': 1}, + {'2022-07-13 12:30:00': '2022-07-13 12:30:00'}, {0.33: 33, 0.67: 67}) + """ + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 1) + assertTrue(result2[0][0] == 1, "Insert should update 1 rows") + } + + try { + sql "DROP TABLE IF EXISTS ${testTable}" + sql "ADMIN SET FRONTEND CONFIG ('enable_map_type' = 'true')" + + create_test_table.call(testTable) + + qt_select "SHOW CREATE TABLE ${testTable}" + } finally { + try_sql("DROP TABLE IF EXISTS ${testTable}") + } + +} From 9dcfbf3ec9f3211770557a0e14d548c01fa46ba3 Mon Sep 17 00:00:00 2001 From: amorynan Date: Fri, 17 Feb 2023 10:47:12 +0800 Subject: [PATCH 4/8] fix regress test case --- regression-test/suites/export/test_map_export.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/export/test_map_export.groovy b/regression-test/suites/export/test_map_export.groovy index 4297d068b7d3f5..fbe94647797d5e 100644 --- a/regression-test/suites/export/test_map_export.groovy +++ b/regression-test/suites/export/test_map_export.groovy @@ -71,7 +71,7 @@ suite("test_map_export", "export") { sql """ INSERT INTO ${testTable} VALUES (2, {}); """ sql """ INSERT INTO ${testTable} VALUES (3, {" 33,amory ":2, " bet ":20, " cler ": 26}); """ sql """ INSERT INTO ${testTable} VALUES (4, {"k3":23, null: 20, "k4": null}); """ - sql """ INSERT INTO ${testTable} VALUES (5, {:2, "k2":}); """ + sql """ INSERT INTO ${testTable} VALUES (5, {null:null}); """ // check result qt_select_default """ SELECT * FROM ${testTable} t ORDER BY k1; """ @@ -119,7 +119,7 @@ suite("test_map_export", "export") { } // check key val empty if (outLine[0] == 5) { - assert outLine[1] == "{:2, \"k2\":}" + assert outLine[1] == "{null:null}" } } } finally { From cc710c0d38d88fe9a41e45392668cab0619d98b3 Mon Sep 17 00:00:00 2001 From: amorynan Date: Fri, 17 Feb 2023 13:17:14 +0800 Subject: [PATCH 5/8] fix map export regress test --- regression-test/data/export/test_map_export.out | 8 ++++++++ regression-test/suites/export/test_map_export.groovy | 10 +++------- 2 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 regression-test/data/export/test_map_export.out diff --git a/regression-test/data/export/test_map_export.out b/regression-test/data/export/test_map_export.out new file mode 100644 index 00000000000000..0e0cfc3f7c9b42 --- /dev/null +++ b/regression-test/data/export/test_map_export.out @@ -0,0 +1,8 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 \N +2 {} +3 {' 33,amory ':2, ' bet ':20, ' cler ':26} +4 {'k3':23, null:20, 'k4':null} +5 {null:null} + diff --git a/regression-test/suites/export/test_map_export.groovy b/regression-test/suites/export/test_map_export.groovy index fbe94647797d5e..9084a0a85dd414 100644 --- a/regression-test/suites/export/test_map_export.groovy +++ b/regression-test/suites/export/test_map_export.groovy @@ -74,21 +74,17 @@ suite("test_map_export", "export") { sql """ INSERT INTO ${testTable} VALUES (5, {null:null}); """ // check result - qt_select_default """ SELECT * FROM ${testTable} t ORDER BY k1; """ + qt_select """ SELECT * FROM ${testTable} ORDER BY id; """ def outFilePath = """${context.file.parent}/tmp""" logger.info("test_map_export the outFilePath=" + outFilePath) // map select into outfile try { File path = new File(outFilePath) - if (path.exists()) { - for (File f : path.listFiles()) { - f.delete(); - } - path.delete(); - } if (!path.exists()) { assert path.mkdirs() + } else { + throw new IllegalStateException("""${outFilePath} already exists! """) } sql """ SELECT * FROM ${testTable} ORDER BY id INTO OUTFILE "file://${outFilePath}/"; From 7c935558141ccffff0dfef35c6c8a3dd7731fd5d Mon Sep 17 00:00:00 2001 From: amorynan Date: Wed, 15 Feb 2023 14:43:07 +0800 Subject: [PATCH 6/8] support stream load and fix some bugs --- be/src/vec/olap/olap_data_convertor.cpp | 6 ++++++ be/src/vec/olap/olap_data_convertor.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp index 431dda727dd337..7f27b31454acc9 100644 --- a/be/src/vec/olap/olap_data_convertor.cpp +++ b/be/src/vec/olap/olap_data_convertor.cpp @@ -786,6 +786,12 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap( return Status::OK(); } +void OlapBlockDataConvertor::OlapColumnDataConvertorMap::set_source_column( + const ColumnWithTypeAndName& typed_column, size_t row_pos, size_t num_rows) { + OlapBlockDataConvertor::OlapColumnDataConvertorBase::set_source_column(typed_column, row_pos, + num_rows); +} + Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap() { const ColumnMap* column_map = nullptr; const DataTypeMap* data_type_map = nullptr; diff --git a/be/src/vec/olap/olap_data_convertor.h b/be/src/vec/olap/olap_data_convertor.h index b23ba5ee5f0bf0..5ec301e5fe876f 100644 --- a/be/src/vec/olap/olap_data_convertor.h +++ b/be/src/vec/olap/olap_data_convertor.h @@ -416,6 +416,8 @@ class OlapBlockDataConvertor { const void* get_data_at(size_t offset) const override { LOG(FATAL) << "now not support get_data_at for OlapColumnDataConvertorMap"; }; + void set_source_column(const ColumnWithTypeAndName& typed_column, size_t row_pos, + size_t num_rows) override; private: Status convert_to_olap(const ColumnMap* column_map, const DataTypeMap* data_type_map); From 84be78dff679695d409531c196cefef04c09c5c8 Mon Sep 17 00:00:00 2001 From: amorynan Date: Thu, 16 Feb 2023 12:52:01 +0800 Subject: [PATCH 7/8] fixed and add some regression test --- regression-test/data/map_p0/test_map.csv | 15 +++ .../map_p0/test_map_load_and_function.out | 37 ++++++ .../suites/map_p0/test_map_dml.groovy | 114 ++++++++++++++++++ .../map_p0/test_map_load_and_function.groovy | 90 ++++++++++++++ 4 files changed, 256 insertions(+) create mode 100644 regression-test/data/map_p0/test_map.csv create mode 100644 regression-test/data/map_p0/test_map_load_and_function.out create mode 100644 regression-test/suites/map_p0/test_map_dml.groovy create mode 100644 regression-test/suites/map_p0/test_map_load_and_function.groovy diff --git a/regression-test/data/map_p0/test_map.csv b/regression-test/data/map_p0/test_map.csv new file mode 100644 index 00000000000000..e7ccbab130cdcc --- /dev/null +++ b/regression-test/data/map_p0/test_map.csv @@ -0,0 +1,15 @@ +1 \N +2 {" 11amory ":23, "beat":20, " clever ": 66} +3 {"k1": 31, "k2": 300} +4 {} +5 \N +6 {"k1":41, "k2": 400} +7 {" 13,amory ":2, " bet ":20, " cler ": 26} +8 {} +9 {' 1,amy ':2, " k2 ":90, " k7 ": 33} +10 {} +11 {"k1': 4, "k2": 400} +12 {"k3":23, null: 20, "k4": null} +13 {"null":1} +15 {:2, "k2":} +16 {null:null} diff --git a/regression-test/data/map_p0/test_map_load_and_function.out b/regression-test/data/map_p0/test_map_load_and_function.out new file mode 100644 index 00000000000000..1f6a5383b5bb30 --- /dev/null +++ b/regression-test/data/map_p0/test_map_load_and_function.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 \N +2 {' 11amory ':23, 'beat':20, ' clever ':66} +3 {'k1':31, 'k2':300} +4 {} +5 \N +6 {'k1':41, 'k2':400} +7 {' 13,amory ':2, ' bet ':20, ' cler ':26} +8 {} +9 {' 1,amy ':2, ' k2 ':90, ' k7 ':33} +10 {} +11 \N +12 {'k3':23, null:20, 'k4':null} +13 {'null':1} +15 {'':2, 'k2':0} +16 {null:null} + +-- !select -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +300 +400 +0 +\N +\N +\N +\N +130 + diff --git a/regression-test/suites/map_p0/test_map_dml.groovy b/regression-test/suites/map_p0/test_map_dml.groovy new file mode 100644 index 00000000000000..65b30a24a52729 --- /dev/null +++ b/regression-test/suites/map_p0/test_map_dml.groovy @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_map_dml", "p0") { + // define a sql table + def testTable = "tbl_test_map_string_int" + def testTable01 = "tbl_test_map_normal" + + def create_test_table = {testTablex -> + def result1 = sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + `k1` INT(11) NULL COMMENT "", + `k2` Map NULL COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2" + ) + """ + + // DDL/DML return 1 row and 3 column, the only value is update row count + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 0, "Create table should update 0 rows") + + // insert 1 row to check whether the table is ok + def result2 = sql "INSERT INTO ${testTable} VALUES (6, {'amory': 6, 'is': 38, 'cl': 0})" + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 1) + assertTrue(result2[0][0] == 1, "Insert should update 1 rows") + } + + def create_test_table01 = {testTablez -> + def result1 = sql """ + CREATE TABLE IF NOT EXISTS ${testTable01} ( + `k1` int(11) NULL, + `k2` Map NULL, + `k3` Map NULL, + `k4` array NULL, + `k5` Map NULL, + `k6` Map NULL, + `k7` Map NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ) + """ + + // DDL/DML return 1 row and 3 column, the only value is update row count + assertTrue(result1.size() == 1) + assertTrue(result1[0].size() == 1) + assertTrue(result1[0][0] == 0, "Create table should update 0 rows") + + + def result2 = sql """ INSERT INTO ${testTable01} VALUES (100, {1: '1', 2: '2', 3:'3'}, {32767: '32767', 32768: '32768', 32769: '32769'}, + [65534, 65535, 65536], {'2022-07-13': 1}, {'2022-07-13 12:30:00': '2022-07-13 12:30:00'}, + {0.33: 33, 0.67: 67}) + """ + + assertTrue(result2.size() == 1) + assertTrue(result2[0].size() == 1) + assertTrue(result2[0][0] == 1, "Insert should update 1 rows") + + qt_test "select * from ${testTable01} order by k1" + } + + + // case1: string_int for map + try { + def res = sql "DROP TABLE IF EXISTS ${testTable}" + + create_test_table.call(testTable) + sql "INSERT INTO ${testTable} VALUES (1, {' amory ': 6, 'happy': 38})" + + // select the table and check whether the data is correct + qt_select "select * from ${testTable} order by k1" + + } finally { + try_sql("DROP TABLE IF EXISTS ${testTable}") + } + + // case2: normal key val type for map + try { + def res = sql "DROP TABLE IF EXISTS ${testTable01}" + create_test_table01.call(testTable) + // select the table and check whether the data is correct + qt_select "select * from ${testTable01} order by k1" + + } finally { + try_sql("DROP TABLE IF EXISTS ${testTable01}") + } +} diff --git a/regression-test/suites/map_p0/test_map_load_and_function.groovy b/regression-test/suites/map_p0/test_map_load_and_function.groovy new file mode 100644 index 00000000000000..b154597ca23d32 --- /dev/null +++ b/regression-test/suites/map_p0/test_map_load_and_function.groovy @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_map_load_and_function", "p0") { + // define a sql table + def testTable = "tbl_test_map" + def dataFile = "test_map.csv" + + sql "DROP TABLE IF EXISTS ${testTable}" + + sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + id INT, + m Map + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES("replication_num" = "1"); + """ + + // load the map data from csv file + streamLoad { + table testTable + + file dataFile // import csv file + time 10000 // limit inflight 10s + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals("OK", json.Message) + assertEquals(15, json.NumberTotalRows) + assertTrue(json.LoadBytes > 0) + + } + } + + // check result + qt_select "SELECT * FROM ${testTable} ORDER BY id" + + // insert into valid json rows + sql """INSERT INTO ${testTable} VALUES(12, NULL)""" + sql """INSERT INTO ${testTable} VALUES(13, {"k1":100, "k2": 130})""" + + // map element_at + qt_select "SELECT m['k2'] FROM ${testTable}" + + // map select into outfile + // check outfile + def outFilePath = """${context.file.parent}/tmp""" + logger.warn("test_map_selectOutFile the outFilePath=" + outFilePath) + + File path = new File(outFilePath) + if (path.exists()) { + for (File f: path.listFiles()) { + f.delete(); + } + path.delete(); + } + if (!path.exists()) { + assert path.mkdirs() + } + sql """ + SELECT * FROM ${testTable} INTO OUTFILE "file://${outFilePath}/"; + """ + File[] files = path.listFiles() + assert files.length == 1 +} From 9b8c433024cb1440557e96ac56f1bd26d948eb44 Mon Sep 17 00:00:00 2001 From: amorynan Date: Thu, 16 Feb 2023 17:54:57 +0800 Subject: [PATCH 8/8] format regression data dir --- be/src/vec/olap/olap_data_convertor.cpp | 6 - be/src/vec/olap/olap_data_convertor.h | 2 - regression-test/data/map_p0/test_map.csv | 15 --- .../map_p0/test_map_load_and_function.out | 37 ------ .../suites/map_p0/test_map_dml.groovy | 114 ------------------ .../map_p0/test_map_load_and_function.groovy | 90 -------------- 6 files changed, 264 deletions(-) delete mode 100644 regression-test/data/map_p0/test_map.csv delete mode 100644 regression-test/data/map_p0/test_map_load_and_function.out delete mode 100644 regression-test/suites/map_p0/test_map_dml.groovy delete mode 100644 regression-test/suites/map_p0/test_map_load_and_function.groovy diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp index 7f27b31454acc9..431dda727dd337 100644 --- a/be/src/vec/olap/olap_data_convertor.cpp +++ b/be/src/vec/olap/olap_data_convertor.cpp @@ -786,12 +786,6 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap( return Status::OK(); } -void OlapBlockDataConvertor::OlapColumnDataConvertorMap::set_source_column( - const ColumnWithTypeAndName& typed_column, size_t row_pos, size_t num_rows) { - OlapBlockDataConvertor::OlapColumnDataConvertorBase::set_source_column(typed_column, row_pos, - num_rows); -} - Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap() { const ColumnMap* column_map = nullptr; const DataTypeMap* data_type_map = nullptr; diff --git a/be/src/vec/olap/olap_data_convertor.h b/be/src/vec/olap/olap_data_convertor.h index 5ec301e5fe876f..b23ba5ee5f0bf0 100644 --- a/be/src/vec/olap/olap_data_convertor.h +++ b/be/src/vec/olap/olap_data_convertor.h @@ -416,8 +416,6 @@ class OlapBlockDataConvertor { const void* get_data_at(size_t offset) const override { LOG(FATAL) << "now not support get_data_at for OlapColumnDataConvertorMap"; }; - void set_source_column(const ColumnWithTypeAndName& typed_column, size_t row_pos, - size_t num_rows) override; private: Status convert_to_olap(const ColumnMap* column_map, const DataTypeMap* data_type_map); diff --git a/regression-test/data/map_p0/test_map.csv b/regression-test/data/map_p0/test_map.csv deleted file mode 100644 index e7ccbab130cdcc..00000000000000 --- a/regression-test/data/map_p0/test_map.csv +++ /dev/null @@ -1,15 +0,0 @@ -1 \N -2 {" 11amory ":23, "beat":20, " clever ": 66} -3 {"k1": 31, "k2": 300} -4 {} -5 \N -6 {"k1":41, "k2": 400} -7 {" 13,amory ":2, " bet ":20, " cler ": 26} -8 {} -9 {' 1,amy ':2, " k2 ":90, " k7 ": 33} -10 {} -11 {"k1': 4, "k2": 400} -12 {"k3":23, null: 20, "k4": null} -13 {"null":1} -15 {:2, "k2":} -16 {null:null} diff --git a/regression-test/data/map_p0/test_map_load_and_function.out b/regression-test/data/map_p0/test_map_load_and_function.out deleted file mode 100644 index 1f6a5383b5bb30..00000000000000 --- a/regression-test/data/map_p0/test_map_load_and_function.out +++ /dev/null @@ -1,37 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select -- -1 \N -2 {' 11amory ':23, 'beat':20, ' clever ':66} -3 {'k1':31, 'k2':300} -4 {} -5 \N -6 {'k1':41, 'k2':400} -7 {' 13,amory ':2, ' bet ':20, ' cler ':26} -8 {} -9 {' 1,amy ':2, ' k2 ':90, ' k7 ':33} -10 {} -11 \N -12 {'k3':23, null:20, 'k4':null} -13 {'null':1} -15 {'':2, 'k2':0} -16 {null:null} - --- !select -- -\N -\N -\N -\N -\N -\N -\N -\N -\N -300 -400 -0 -\N -\N -\N -\N -130 - diff --git a/regression-test/suites/map_p0/test_map_dml.groovy b/regression-test/suites/map_p0/test_map_dml.groovy deleted file mode 100644 index 65b30a24a52729..00000000000000 --- a/regression-test/suites/map_p0/test_map_dml.groovy +++ /dev/null @@ -1,114 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_map_dml", "p0") { - // define a sql table - def testTable = "tbl_test_map_string_int" - def testTable01 = "tbl_test_map_normal" - - def create_test_table = {testTablex -> - def result1 = sql """ - CREATE TABLE IF NOT EXISTS ${testTable} ( - `k1` INT(11) NULL COMMENT "", - `k2` Map NULL COMMENT "" - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - DISTRIBUTED BY HASH(`k1`) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "storage_format" = "V2" - ) - """ - - // DDL/DML return 1 row and 3 column, the only value is update row count - assertTrue(result1.size() == 1) - assertTrue(result1[0].size() == 1) - assertTrue(result1[0][0] == 0, "Create table should update 0 rows") - - // insert 1 row to check whether the table is ok - def result2 = sql "INSERT INTO ${testTable} VALUES (6, {'amory': 6, 'is': 38, 'cl': 0})" - assertTrue(result2.size() == 1) - assertTrue(result2[0].size() == 1) - assertTrue(result2[0][0] == 1, "Insert should update 1 rows") - } - - def create_test_table01 = {testTablez -> - def result1 = sql """ - CREATE TABLE IF NOT EXISTS ${testTable01} ( - `k1` int(11) NULL, - `k2` Map NULL, - `k3` Map NULL, - `k4` array NULL, - `k5` Map NULL, - `k6` Map NULL, - `k7` Map NULL - ) ENGINE=OLAP - DUPLICATE KEY(`k1`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`k1`) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "in_memory" = "false", - "storage_format" = "V2", - "disable_auto_compaction" = "false" - ) - """ - - // DDL/DML return 1 row and 3 column, the only value is update row count - assertTrue(result1.size() == 1) - assertTrue(result1[0].size() == 1) - assertTrue(result1[0][0] == 0, "Create table should update 0 rows") - - - def result2 = sql """ INSERT INTO ${testTable01} VALUES (100, {1: '1', 2: '2', 3:'3'}, {32767: '32767', 32768: '32768', 32769: '32769'}, - [65534, 65535, 65536], {'2022-07-13': 1}, {'2022-07-13 12:30:00': '2022-07-13 12:30:00'}, - {0.33: 33, 0.67: 67}) - """ - - assertTrue(result2.size() == 1) - assertTrue(result2[0].size() == 1) - assertTrue(result2[0][0] == 1, "Insert should update 1 rows") - - qt_test "select * from ${testTable01} order by k1" - } - - - // case1: string_int for map - try { - def res = sql "DROP TABLE IF EXISTS ${testTable}" - - create_test_table.call(testTable) - sql "INSERT INTO ${testTable} VALUES (1, {' amory ': 6, 'happy': 38})" - - // select the table and check whether the data is correct - qt_select "select * from ${testTable} order by k1" - - } finally { - try_sql("DROP TABLE IF EXISTS ${testTable}") - } - - // case2: normal key val type for map - try { - def res = sql "DROP TABLE IF EXISTS ${testTable01}" - create_test_table01.call(testTable) - // select the table and check whether the data is correct - qt_select "select * from ${testTable01} order by k1" - - } finally { - try_sql("DROP TABLE IF EXISTS ${testTable01}") - } -} diff --git a/regression-test/suites/map_p0/test_map_load_and_function.groovy b/regression-test/suites/map_p0/test_map_load_and_function.groovy deleted file mode 100644 index b154597ca23d32..00000000000000 --- a/regression-test/suites/map_p0/test_map_load_and_function.groovy +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.codehaus.groovy.runtime.IOGroovyMethods - -suite("test_map_load_and_function", "p0") { - // define a sql table - def testTable = "tbl_test_map" - def dataFile = "test_map.csv" - - sql "DROP TABLE IF EXISTS ${testTable}" - - sql """ - CREATE TABLE IF NOT EXISTS ${testTable} ( - id INT, - m Map - ) - DUPLICATE KEY(id) - DISTRIBUTED BY HASH(id) BUCKETS 10 - PROPERTIES("replication_num" = "1"); - """ - - // load the map data from csv file - streamLoad { - table testTable - - file dataFile // import csv file - time 10000 // limit inflight 10s - - // if declared a check callback, the default check condition will ignore. - // So you must check all condition - check { result, exception, startTime, endTime -> - if (exception != null) { - throw exception - } - log.info("Stream load result: ${result}".toString()) - def json = parseJson(result) - assertEquals("success", json.Status.toLowerCase()) - assertEquals("OK", json.Message) - assertEquals(15, json.NumberTotalRows) - assertTrue(json.LoadBytes > 0) - - } - } - - // check result - qt_select "SELECT * FROM ${testTable} ORDER BY id" - - // insert into valid json rows - sql """INSERT INTO ${testTable} VALUES(12, NULL)""" - sql """INSERT INTO ${testTable} VALUES(13, {"k1":100, "k2": 130})""" - - // map element_at - qt_select "SELECT m['k2'] FROM ${testTable}" - - // map select into outfile - // check outfile - def outFilePath = """${context.file.parent}/tmp""" - logger.warn("test_map_selectOutFile the outFilePath=" + outFilePath) - - File path = new File(outFilePath) - if (path.exists()) { - for (File f: path.listFiles()) { - f.delete(); - } - path.delete(); - } - if (!path.exists()) { - assert path.mkdirs() - } - sql """ - SELECT * FROM ${testTable} INTO OUTFILE "file://${outFilePath}/"; - """ - File[] files = path.listFiles() - assert files.length == 1 -}