Skip to content

[Fix](Lightweight schema Change) query error caused by array default type is unsupported #17331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 5 commits into from
Mar 7, 2023
Merged

[Fix](Lightweight schema Change) query error caused by array default type is unsupported #17331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
86e7402
fix
ZhangYu0123 Mar 1, 2023
5ba4ed4
fix
ZhangYu0123 Mar 2, 2023
f432075
format
ZhangYu0123 Mar 2, 2023
df69db8
add regression test
ZhangYu0123 Mar 3, 2023
172f476
fix
ZhangYu0123 Mar 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 37 additions & 12 deletions be/src/olap/rowset/segment_v2/column_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1117,8 +1117,16 @@ Status DefaultValueColumnIterator::init(const ColumnIteratorOptions& opts) {
((Slice*)_mem_value.data())->size = _default_value.length();
((Slice*)_mem_value.data())->data = _default_value.data();
} else if (_type_info->type() == OLAP_FIELD_TYPE_ARRAY) {
// TODO llj for Array default value
return Status::NotSupported("Array default type is unsupported");
if (_default_value != "[]") {
return Status::NotSupported("Array default {} is unsupported", _default_value);
} else {
((Slice*)_mem_value.data())->size = _default_value.length();
((Slice*)_mem_value.data())->data = _default_value.data();
}
} else if (_type_info->type() == OLAP_FIELD_TYPE_STRUCT) {
return Status::NotSupported("STRUCT default type is unsupported");
} else if (_type_info->type() == OLAP_FIELD_TYPE_MAP) {
return Status::NotSupported("MAP default type is unsupported");
} else {
s = _type_info->from_string(_mem_value.data(), _default_value, _precision, _scale);
}
Expand All @@ -1139,9 +1147,6 @@ Status DefaultValueColumnIterator::init(const ColumnIteratorOptions& opts) {
void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info, size_t type_size,
void* mem_value,
vectorized::MutableColumnPtr& dst, size_t n) {
vectorized::Int128 int128;
char* data_ptr = (char*)&int128;
size_t data_len = sizeof(int128);
dst = dst->convert_to_predicate_column_if_dictionary();

switch (type_info->type()) {
Expand All @@ -1151,30 +1156,42 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info,
break;
}
case OLAP_FIELD_TYPE_DATE: {
vectorized::Int64 int64;
char* data_ptr = (char*)&int64;
size_t data_len = sizeof(int64);

assert(type_size == sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DATE>::CppType)); //uint24_t
std::string str = FieldTypeTraits<OLAP_FIELD_TYPE_DATE>::to_string(mem_value);

vectorized::VecDateTimeValue value;
value.from_date_str(str.c_str(), str.length());
value.cast_to_date();
//TODO: here is int128 = int64, here rely on the logic of little endian
int128 = binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(value);

int64 = binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(value);
dst->insert_many_data(data_ptr, data_len, n);
break;
}
case OLAP_FIELD_TYPE_DATETIME: {
vectorized::Int64 int64;
char* data_ptr = (char*)&int64;
size_t data_len = sizeof(int64);

assert(type_size == sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DATETIME>::CppType)); //int64_t
std::string str = FieldTypeTraits<OLAP_FIELD_TYPE_DATETIME>::to_string(mem_value);

vectorized::VecDateTimeValue value;
value.from_date_str(str.c_str(), str.length());
value.to_datetime();

int128 = binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(value);
int64 = binary_cast<vectorized::VecDateTimeValue, vectorized::Int64>(value);
dst->insert_many_data(data_ptr, data_len, n);
break;
}
case OLAP_FIELD_TYPE_DECIMAL: {
vectorized::Int128 int128;
char* data_ptr = (char*)&int128;
size_t data_len = sizeof(int128);

assert(type_size ==
sizeof(FieldTypeTraits<OLAP_FIELD_TYPE_DECIMAL>::CppType)); //decimal12_t
decimal12_t* d = (decimal12_t*)mem_value;
Expand All @@ -1186,14 +1203,22 @@ void DefaultValueColumnIterator::insert_default_data(const TypeInfo* type_info,
case OLAP_FIELD_TYPE_VARCHAR:
case OLAP_FIELD_TYPE_CHAR:
case OLAP_FIELD_TYPE_JSONB: {
data_ptr = ((Slice*)mem_value)->data;
data_len = ((Slice*)mem_value)->size;
char* data_ptr = ((Slice*)mem_value)->data;
size_t data_len = ((Slice*)mem_value)->size;
dst->insert_many_data(data_ptr, data_len, n);
break;
}
case OLAP_FIELD_TYPE_ARRAY: {
if (dst->is_nullable()) {
static_cast<vectorized::ColumnNullable&>(*dst).insert_not_null_elements(n);
} else {
dst->insert_many_defaults(n);
}
break;
}
default: {
data_ptr = (char*)mem_value;
data_len = type_size;
char* data_ptr = (char*)mem_value;
size_t data_len = type_size;
dst->insert_many_data(data_ptr, data_len, n);
}
}
Expand Down
6 changes: 6 additions & 0 deletions be/src/vec/columns/column_nullable.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,12 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable> {
_has_null = true;
}

void insert_not_null_elements(size_t num) {
get_nested_column().insert_many_defaults(num);
_get_null_map_column().fill(0, num);
_has_null = false;
}

void insert_null_elements(int num) {
get_nested_column().insert_many_defaults(num);
_get_null_map_column().fill(1, num);
Expand Down
11 changes: 11 additions & 0 deletions regression-test/data/schema_change_p0/test_alter_table_column.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,17 @@ value2 INT Yes false \N SUM
-- !sql --
1 1 40 60

-- !sql --
k1 INT Yes true \N
value1 INT Yes false \N NONE
value2 ARRAY<INT(11)> Yes false [] NONE
value3 ARRAY<INT(11)> Yes false \N NONE
value4 ARRAY<INT(11)> No false [] NONE

-- !sql --
1 2 [] \N []
3 4 [] \N []

-- !select --
\N \N \N
1 1989 1001
Expand Down
42 changes: 42 additions & 0 deletions regression-test/suites/schema_change_p0/test_alter_table_column.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,47 @@ suite("test_alter_table_column") {
qt_sql "select * from ${tbName2};"
sql "DROP TABLE ${tbName2} FORCE;"

def tbNameAddArray = "alter_table_add_array_column_dup"
sql "DROP TABLE IF EXISTS ${tbNameAddArray}"
sql """
CREATE TABLE IF NOT EXISTS ${tbNameAddArray} (
k1 INT,
value1 INT
)
DUPLICATE KEY (k1)
DISTRIBUTED BY HASH(k1) BUCKETS 5 properties(
"replication_num" = "1",
"light_schema_change" = "true",
"disable_auto_compaction" = "true");
"""

sql "insert into ${tbNameAddArray} values(1,2)"
sql "insert into ${tbNameAddArray} values(3,4)"
sql """
ALTER TABLE ${tbNameAddArray}
ADD COLUMN value2 ARRAY<INT> DEFAULT '[]' AFTER value1,
ADD COLUMN value3 ARRAY<INT> AFTER value2,
ADD COLUMN value4 ARRAY<INT> NOT NULL DEFAULT '[]' AFTER value3;
"""
max_try_secs = 60
while (max_try_secs--) {
String res = getJobState(tbNameAddArray)
if (res == "FINISHED") {
break
} else {
Thread.sleep(2000)
if (max_try_secs < 1) {
println "test timeout," + "state:" + res
assertEquals("FINISHED",res)
}
}
}

Thread.sleep(200)
qt_sql "desc ${tbNameAddArray};"
qt_sql "select * from ${tbNameAddArray};"
sql "DROP TABLE ${tbNameAddArray} FORCE;"

// vector search
def check_load_result = {checklabel, testTablex ->
Integer max_try_milli_secs = 10000
Expand Down Expand Up @@ -175,4 +216,5 @@ suite("test_alter_table_column") {
def res4 = sql "select k1, k2, k3, null from baseall order by k1"
check2_doris(res3, res4)
sql "DROP TABLE ${tbName3} FORCE;"

}