From 86e740284909f65b030026862bc19b1bb745c2ce Mon Sep 17 00:00:00 2001 From: ZhangYu0123 Date: Wed, 1 Mar 2023 22:05:20 +0800 Subject: [PATCH 1/5] fix --- .../olap/rowset/segment_v2/column_reader.cpp | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 9079118ecd18a6..4adb4ec970611d 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -1139,9 +1139,6 @@ Status DefaultValueColumnIterator::init(const ColumnIteratorOptions& opts) { void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info, size_t type_size, void* mem_value, vectorized::MutableColumnPtr& dst, size_t n) { - vectorized::Int128 int128; - char* data_ptr = (char*)&int128; - size_t data_len = sizeof(int128); dst = dst->convert_to_predicate_column_if_dictionary(); switch (type_info->type()) { @@ -1151,18 +1148,26 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info, break; } case OLAP_FIELD_TYPE_DATE: { + vectorized::Int64 int64; + char* data_ptr = (char*)&int64; + size_t data_len = sizeof(int64); + assert(type_size == sizeof(FieldTypeTraits::CppType)); //uint24_t std::string str = FieldTypeTraits::to_string(mem_value); vectorized::VecDateTimeValue value; value.from_date_str(str.c_str(), str.length()); value.cast_to_date(); - //TODO: here is int128 = int64, here rely on the logic of little endian - int128 = binary_cast(value); + + int64 = binary_cast(value); dst->insert_many_data(data_ptr, data_len, n); break; } case OLAP_FIELD_TYPE_DATETIME: { + vectorized::Int64 int64; + char* data_ptr = (char*)&int64; + size_t data_len = sizeof(int64); + assert(type_size == sizeof(FieldTypeTraits::CppType)); //int64_t std::string str = FieldTypeTraits::to_string(mem_value); @@ -1170,11 +1175,15 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info, value.from_date_str(str.c_str(), str.length()); value.to_datetime(); - int128 = binary_cast(value); + int64 = binary_cast(value); dst->insert_many_data(data_ptr, data_len, n); break; } case OLAP_FIELD_TYPE_DECIMAL: { + vectorized::Int128 int128; + char* data_ptr = (char*)&int128; + size_t data_len = sizeof(int128); + assert(type_size == sizeof(FieldTypeTraits::CppType)); //decimal12_t decimal12_t* d = (decimal12_t*)mem_value; @@ -1186,14 +1195,14 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info, case OLAP_FIELD_TYPE_VARCHAR: case OLAP_FIELD_TYPE_CHAR: case OLAP_FIELD_TYPE_JSONB: { - data_ptr = ((Slice*)mem_value)->data; - data_len = ((Slice*)mem_value)->size; + char* data_ptr = ((Slice*)mem_value)->data; + size_t data_len = ((Slice*)mem_value)->size; dst->insert_many_data(data_ptr, data_len, n); break; } default: { - data_ptr = (char*)mem_value; - data_len = type_size; + char* data_ptr = (char*)mem_value; + size_t data_len = type_size; dst->insert_many_data(data_ptr, data_len, n); } } From 5ba4ed48ed3dc316b47eed7afbcd9190825c7a1a Mon Sep 17 00:00:00 2001 From: ZhangYu0123 Date: Thu, 2 Mar 2023 14:48:36 +0800 Subject: [PATCH 2/5] fix --- be/src/olap/rowset/segment_v2/column_reader.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 4adb4ec970611d..b772bc02dff88a 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -1117,8 +1117,16 @@ Status DefaultValueColumnIterator::init(const ColumnIteratorOptions& opts) { ((Slice*)_mem_value.data())->size = _default_value.length(); ((Slice*)_mem_value.data())->data = _default_value.data(); } else if (_type_info->type() == OLAP_FIELD_TYPE_ARRAY) { - // TODO llj for Array default value - return Status::NotSupported("Array default type is unsupported"); + if (_default_value != "[]") { + return Status::NotSupported("Array default {} is unsupported", _default_value); + } else { + ((Slice*)_mem_value.data())->size = _default_value.length(); + ((Slice*)_mem_value.data())->data = _default_value.data(); + } + } else if (_type_info->type() == OLAP_FIELD_TYPE_STRUCT) { + return Status::NotSupported("STRUCT default type is unsupported"); + } else if (_type_info->type() == OLAP_FIELD_TYPE_MAP) { + return Status::NotSupported("MAP default type is unsupported"); } else { s = _type_info->from_string(_mem_value.data(), _default_value, _precision, _scale); } @@ -1200,6 +1208,10 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info, dst->insert_many_data(data_ptr, data_len, n); break; } + case OLAP_FIELD_TYPE_ARRAY: { + dst->insert_many_defaults(n); + break; + } default: { char* data_ptr = (char*)mem_value; size_t data_len = type_size; From f432075c9536d969d23b50a18c0432085d5a92a5 Mon Sep 17 00:00:00 2001 From: ZhangYu0123 Date: Thu, 2 Mar 2023 15:09:24 +0800 Subject: [PATCH 3/5] format --- be/src/olap/rowset/segment_v2/column_reader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index b772bc02dff88a..52085d64d76d66 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -1166,7 +1166,7 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info, vectorized::VecDateTimeValue value; value.from_date_str(str.c_str(), str.length()); value.cast_to_date(); - + int64 = binary_cast(value); dst->insert_many_data(data_ptr, data_len, n); break; From df69db8cb2b740ebef2252d7ba423ae2b96c26a3 Mon Sep 17 00:00:00 2001 From: ZhangYu0123 Date: Fri, 3 Mar 2023 23:17:38 +0800 Subject: [PATCH 4/5] add regression test --- .../olap/rowset/segment_v2/column_reader.cpp | 5 ++- .../test_alter_table_column.out | 10 +++++ .../test_alter_table_column.groovy | 41 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 52085d64d76d66..40766190f89795 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -32,6 +32,7 @@ #include "vec/columns/column_array.h" #include "vec/columns/column_map.h" #include "vec/columns/column_struct.h" +#include "vec/core/field.h" #include "vec/core/types.h" #include "vec/runtime/vdatetime_value.h" //for VecDateTime @@ -1209,7 +1210,9 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info, break; } case OLAP_FIELD_TYPE_ARRAY: { - dst->insert_many_defaults(n); + for (int i = 0; i < n; i++) { + dst->insert(doris::vectorized::Array(0)); + } break; } default: { diff --git a/regression-test/data/schema_change_p0/test_alter_table_column.out b/regression-test/data/schema_change_p0/test_alter_table_column.out index a130d16e16e71b..19eb93b359f48b 100644 --- a/regression-test/data/schema_change_p0/test_alter_table_column.out +++ b/regression-test/data/schema_change_p0/test_alter_table_column.out @@ -18,6 +18,16 @@ value2 INT Yes false \N SUM -- !sql -- 1 1 40 60 +-- !sql -- +k1 INT Yes true \N +value1 INT Yes false \N NONE +value2 ARRAY Yes false [] NONE +value3 ARRAY Yes false \N NONE + +-- !sql -- +1 2 [] \N +3 4 [] \N + -- !select -- \N \N \N 1 1989 1001 diff --git a/regression-test/suites/schema_change_p0/test_alter_table_column.groovy b/regression-test/suites/schema_change_p0/test_alter_table_column.groovy index e6848dc7fefec6..8fc705eaafecd2 100644 --- a/regression-test/suites/schema_change_p0/test_alter_table_column.groovy +++ b/regression-test/suites/schema_change_p0/test_alter_table_column.groovy @@ -114,6 +114,46 @@ suite("test_alter_table_column") { qt_sql "select * from ${tbName2};" sql "DROP TABLE ${tbName2} FORCE;" + def tbNameAddArray = "alter_table_add_array_column_dup" + sql "DROP TABLE IF EXISTS ${tbNameAddArray}" + sql """ + CREATE TABLE IF NOT EXISTS ${tbNameAddArray} ( + k1 INT, + value1 INT + ) + DUPLICATE KEY (k1) + DISTRIBUTED BY HASH(k1) BUCKETS 5 properties( + "replication_num" = "1", + "light_schema_change" = "true", + "disable_auto_compaction" = "true"); + """ + + sql "insert into ${tbNameAddArray} values(1,2)" + sql "insert into ${tbNameAddArray} values(3,4)" + sql """ + ALTER TABLE ${tbNameAddArray} + ADD COLUMN value2 ARRAY DEFAULT '[]' AFTER value1, + ADD COLUMN value3 ARRAY AFTER value2 + """ + max_try_secs = 60 + while (max_try_secs--) { + String res = getJobState(tbNameAddArray) + if (res == "FINISHED") { + break + } else { + Thread.sleep(2000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED",res) + } + } + } + + Thread.sleep(200) + qt_sql "desc ${tbNameAddArray};" + qt_sql "select * from ${tbNameAddArray};" + sql "DROP TABLE ${tbNameAddArray} FORCE;" + // vector search def check_load_result = {checklabel, testTablex -> Integer max_try_milli_secs = 10000 @@ -175,4 +215,5 @@ suite("test_alter_table_column") { def res4 = sql "select k1, k2, k3, null from baseall order by k1" check2_doris(res3, res4) sql "DROP TABLE ${tbName3} FORCE;" + } From 172f4760e1822965c483f42289d5f08491b32f93 Mon Sep 17 00:00:00 2001 From: ZhangYu0123 Date: Sat, 4 Mar 2023 12:22:54 +0800 Subject: [PATCH 5/5] fix --- be/src/olap/rowset/segment_v2/column_reader.cpp | 7 ++++--- be/src/vec/columns/column_nullable.h | 6 ++++++ .../data/schema_change_p0/test_alter_table_column.out | 5 +++-- .../suites/schema_change_p0/test_alter_table_column.groovy | 3 ++- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 40766190f89795..6cd0333028a01c 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -32,7 +32,6 @@ #include "vec/columns/column_array.h" #include "vec/columns/column_map.h" #include "vec/columns/column_struct.h" -#include "vec/core/field.h" #include "vec/core/types.h" #include "vec/runtime/vdatetime_value.h" //for VecDateTime @@ -1210,8 +1209,10 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info, break; } case OLAP_FIELD_TYPE_ARRAY: { - for (int i = 0; i < n; i++) { - dst->insert(doris::vectorized::Array(0)); + if (dst->is_nullable()) { + static_cast(*dst).insert_not_null_elements(n); + } else { + dst->insert_many_defaults(n); } break; } diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h index c48e3c80a18eed..d9216cde49e9d6 100644 --- a/be/src/vec/columns/column_nullable.h +++ b/be/src/vec/columns/column_nullable.h @@ -153,6 +153,12 @@ class ColumnNullable final : public COWHelper { _has_null = true; } + void insert_not_null_elements(size_t num) { + get_nested_column().insert_many_defaults(num); + _get_null_map_column().fill(0, num); + _has_null = false; + } + void insert_null_elements(int num) { get_nested_column().insert_many_defaults(num); _get_null_map_column().fill(1, num); diff --git a/regression-test/data/schema_change_p0/test_alter_table_column.out b/regression-test/data/schema_change_p0/test_alter_table_column.out index 19eb93b359f48b..81a715c2bcdd22 100644 --- a/regression-test/data/schema_change_p0/test_alter_table_column.out +++ b/regression-test/data/schema_change_p0/test_alter_table_column.out @@ -23,10 +23,11 @@ k1 INT Yes true \N value1 INT Yes false \N NONE value2 ARRAY Yes false [] NONE value3 ARRAY Yes false \N NONE +value4 ARRAY No false [] NONE -- !sql -- -1 2 [] \N -3 4 [] \N +1 2 [] \N [] +3 4 [] \N [] -- !select -- \N \N \N diff --git a/regression-test/suites/schema_change_p0/test_alter_table_column.groovy b/regression-test/suites/schema_change_p0/test_alter_table_column.groovy index 8fc705eaafecd2..56fb531074935b 100644 --- a/regression-test/suites/schema_change_p0/test_alter_table_column.groovy +++ b/regression-test/suites/schema_change_p0/test_alter_table_column.groovy @@ -133,7 +133,8 @@ suite("test_alter_table_column") { sql """ ALTER TABLE ${tbNameAddArray} ADD COLUMN value2 ARRAY DEFAULT '[]' AFTER value1, - ADD COLUMN value3 ARRAY AFTER value2 + ADD COLUMN value3 ARRAY AFTER value2, + ADD COLUMN value4 ARRAY NOT NULL DEFAULT '[]' AFTER value3; """ max_try_secs = 60 while (max_try_secs--) {