Skip to content

[Schema change] Support More column type in schema change #4938

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 2 commits into from
Nov 28, 2020
Merged

[Schema change] Support More column type in schema change #4938

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
08f228b
[Schema change] This pr of 2 mainly change
HappenLee Nov 23, 2020
5137dec
change indent and remove some code
HappenLee Nov 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 17 additions & 37 deletions be/src/olap/schema_change.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ ColumnMapping* RowBlockChanger::get_mutable_column_mapping(size_t column_index)

#define CONVERT_FROM_TYPE(from_type) \
{ \
switch (mutable_block->tablet_schema().column(i).type()) { \
switch (newtype) { \
case OLAP_FIELD_TYPE_TINYINT: \
TYPE_REINTERPRET_CAST(from_type, int8_t); \
case OLAP_FIELD_TYPE_UNSIGNED_TINYINT: \
Expand All @@ -168,6 +168,8 @@ ColumnMapping* RowBlockChanger::get_mutable_column_mapping(size_t column_index)
TYPE_REINTERPRET_CAST(from_type, uint64_t); \
case OLAP_FIELD_TYPE_LARGEINT: \
LARGEINT_REINTERPRET_CAST(from_type, int128_t); \
case OLAP_FIELD_TYPE_FLOAT: \
TYPE_REINTERPRET_CAST(from_type, float); \
case OLAP_FIELD_TYPE_DOUBLE: \
TYPE_REINTERPRET_CAST(from_type, double); \
default: \
Expand Down Expand Up @@ -217,6 +219,16 @@ class ConvertTypeResolver {
};

ConvertTypeResolver::ConvertTypeResolver() {
// from char type
add_convert_type_mapping<OLAP_FIELD_TYPE_CHAR, OLAP_FIELD_TYPE_TINYINT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_CHAR, OLAP_FIELD_TYPE_SMALLINT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_CHAR, OLAP_FIELD_TYPE_INT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_CHAR, OLAP_FIELD_TYPE_BIGINT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_CHAR, OLAP_FIELD_TYPE_LARGEINT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_CHAR, OLAP_FIELD_TYPE_FLOAT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_CHAR, OLAP_FIELD_TYPE_DOUBLE>();
add_convert_type_mapping<OLAP_FIELD_TYPE_CHAR, OLAP_FIELD_TYPE_DATE>();

// supported type convert should annotate in doc:
// http://doris.incubator.apache.org/master/zh-CN/sql-reference/sql-statements/Data%20Definition/ALTER%20TABLE.html#description
// If type convert is supported here, you should check fe/src/main/java/org/apache/doris/catalog/ColumnType.java to supported it either
Expand All @@ -239,6 +251,7 @@ ConvertTypeResolver::ConvertTypeResolver() {
add_convert_type_mapping<OLAP_FIELD_TYPE_FLOAT, OLAP_FIELD_TYPE_VARCHAR>();
add_convert_type_mapping<OLAP_FIELD_TYPE_DOUBLE, OLAP_FIELD_TYPE_VARCHAR>();
add_convert_type_mapping<OLAP_FIELD_TYPE_DECIMAL, OLAP_FIELD_TYPE_VARCHAR>();
add_convert_type_mapping<OLAP_FIELD_TYPE_CHAR, OLAP_FIELD_TYPE_VARCHAR>();

add_convert_type_mapping<OLAP_FIELD_TYPE_DATE, OLAP_FIELD_TYPE_DATETIME>();

Expand Down Expand Up @@ -452,7 +465,6 @@ OLAPStatus RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t
for (size_t i = 0, len = mutable_block->tablet_schema().num_columns(); !filter_all && i < len;
++i) {
int32_t ref_column = _schema_mapping[i].ref_column;

if (_schema_mapping[i].ref_column >= 0) {
if (!_schema_mapping[i].materialized_function.empty()) {
bool (*_do_materialized_transform) (RowCursor*, RowCursor*, const TabletColumn&, int, int, MemPool* );
Expand Down Expand Up @@ -523,38 +535,6 @@ OLAPStatus RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t
}
}
}

// 从ref_column 写入 i列。
} else if (newtype == OLAP_FIELD_TYPE_VARCHAR && reftype == OLAP_FIELD_TYPE_CHAR) {
// 效率低下，也可以直接计算变长域拷贝，但仍然会破坏封装
for (size_t row_index = 0, new_row_index = 0;
row_index < ref_block->row_block_info().row_num; ++row_index) {
// 不需要的row，每次处理到这个row时就跳过
if (need_filter_data && is_data_left_vec[row_index] == 0) {
continue;
}

// 指定新的要写入的row index（不同于读的row_index）
mutable_block->get_row(new_row_index++, &write_helper);

ref_block->get_row(row_index, &read_helper);

if (true == read_helper.is_null(ref_column)) {
write_helper.set_null(i);
} else {
// 要写入的

write_helper.set_not_null(i);
int p = ref_block->tablet_schema().column(ref_column).length() - 1;
Slice* slice = reinterpret_cast<Slice*>(read_helper.cell_ptr(ref_column));
char* buf = slice->data;
while (p >= 0 && buf[p] == '\0') {
p--;
}
slice->size = p + 1;
write_helper.set_field_content(i, reinterpret_cast<char*>(slice), mem_pool);
}
}
} else if (ConvertTypeResolver::instance()->get_convert_type_info(reftype, newtype)) {
for (size_t row_index = 0, new_row_index = 0;
row_index < ref_block->row_block_info().row_num; ++row_index) {
Expand Down Expand Up @@ -602,6 +582,8 @@ OLAPStatus RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t
CONVERT_FROM_TYPE(int64_t);
case OLAP_FIELD_TYPE_UNSIGNED_BIGINT:
CONVERT_FROM_TYPE(uint64_t);
case OLAP_FIELD_TYPE_LARGEINT:
CONVERT_FROM_TYPE(int128_t);
default:
LOG(WARNING) << "the column type which was altered from was unsupported."
<< " from_type="
Expand Down Expand Up @@ -736,10 +718,8 @@ bool RowBlockSorter::sort(RowBlock** row_block) {
RowBlockAllocator::RowBlockAllocator(const TabletSchema& tablet_schema, size_t memory_limitation)
: _tablet_schema(tablet_schema),
_memory_allocated(0),
_row_len(tablet_schema.row_size()),
_memory_limitation(memory_limitation) {
_row_len = 0;
_row_len = tablet_schema.row_size();

VLOG(3) << "RowBlockAllocator(). row_len=" << _row_len;
}

Expand Down
152 changes: 61 additions & 91 deletions be/src/olap/types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "gen_cpp/segment_v2.pb.h" // for ColumnMetaPB
#include "olap/collection.h"
#include "olap/decimal12.h"

#include "olap/olap_common.h"
#include "olap/olap_define.h"
#include "olap/tablet_schema.h" // for TabletColumn
Expand Down Expand Up @@ -495,12 +496,32 @@ struct BaseFieldtypeTraits : public CppTypeTraits<field_type> {
}
};

static void prepare_char_before_convert(const void* src) {
Slice* slice = const_cast<Slice*>(reinterpret_cast<const Slice*>(src));
char* buf = slice->data;
auto p = slice->size - 1;
while (p >= 0 && buf[p] == '\0') {
p--;
}
slice->size = p + 1;
}

template <typename T>
OLAPStatus convert_int_from_varchar(void* dest, const void* src) {
using SrcType = typename CppTypeTraits<OLAP_FIELD_TYPE_VARCHAR>::CppType;
auto src_value = reinterpret_cast<const SrcType*>(src);
T convert_from_varchar(const Slice* src_value, StringParser::ParseResult& parse_res, std::true_type) {
return StringParser::string_to_int<T>(src_value->get_data(), src_value->get_size(), &parse_res);
}

template <typename T>
Copy link
Contributor

@chaoyli chaoyli Nov 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There may use traits ?
Or use if constexpr (std::is_floating_point_v)) to distinguish the two cases.

Copy link
Contributor Author

@HappenLee HappenLee Nov 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi, chaoyli.

if constexpr only support in c++17 and std::is_floating_point_v support in c++14.

Now, Doris only support C++11, So I use label assignment to distinguish the float and int.

T convert_from_varchar(const Slice* src_value, StringParser::ParseResult& parse_res, std::false_type) {
return StringParser::string_to_float<T>(src_value->get_data(), src_value->get_size(), &parse_res);
}

template <typename T>
OLAPStatus arithmetic_convert_from_varchar(void* dest, const void* src) {
auto src_value = reinterpret_cast<const Slice*>(src);
StringParser::ParseResult parse_res;
T result = StringParser::string_to_int<T>(src_value->get_data(), src_value->get_size(), &parse_res);
//TODO: use C++17 if constexpr to replace label assignment
auto result = convert_from_varchar<T>(src_value, parse_res, std::is_integral<T>());
if (UNLIKELY(parse_res != StringParser::PARSE_SUCCESS)) {
return OLAPStatus::OLAP_ERR_INVALID_SCHEMA;
}
Expand All @@ -509,98 +530,55 @@ OLAPStatus convert_int_from_varchar(void* dest, const void* src) {
}

template <typename T>
OLAPStatus convert_float_from_varchar(void* dest, const void* src) {
using SrcType = typename CppTypeTraits<OLAP_FIELD_TYPE_VARCHAR>::CppType;
auto src_value = reinterpret_cast<const SrcType *>(src);
StringParser::ParseResult parse_res;
T result = StringParser::string_to_float<T>(src_value->get_data(), src_value->get_size(), &parse_res);
if (UNLIKELY(parse_res != StringParser::PARSE_SUCCESS)) {
return OLAPStatus::OLAP_ERR_INVALID_SCHEMA;
}
*reinterpret_cast<T*>(dest) = result;
return OLAPStatus::OLAP_SUCCESS;
OLAPStatus numeric_convert_from_char(void *dest, const void *src) {
prepare_char_before_convert(src);
return arithmetic_convert_from_varchar<T>(dest, src);
}

template<FieldType field_type>
struct FieldTypeTraits : public BaseFieldtypeTraits<field_type> { };
// Using NumericFieldtypeTraits to Derived code for OLAP_FIELD_TYPE_XXXINT, OLAP_FIELD_TYPE_FLOAT,
// OLAP_FIELD_TYPE_DOUBLE, to reduce redundant code
template <FieldType fieldType, bool isArithmetic>
struct NumericFieldtypeTraits : public BaseFieldtypeTraits<fieldType> {
using CppType = typename CppTypeTraits<fieldType>::CppType;

template<>
struct FieldTypeTraits<OLAP_FIELD_TYPE_BOOL> : public BaseFieldtypeTraits<OLAP_FIELD_TYPE_BOOL> {
static std::string to_string(const void* src) {
char buf[1024] = {'\0'};
snprintf(buf, sizeof(buf), "%d", *reinterpret_cast<const bool*>(src));
return std::string(buf);
return std::to_string(*reinterpret_cast<const CppType*>(src));
}
static void set_to_max(void* buf) {
(*(bool*)buf) = true;
}
static void set_to_min(void* buf) {
(*(bool*)buf) = false;
}
};

template<>
struct FieldTypeTraits<OLAP_FIELD_TYPE_TINYINT> : public BaseFieldtypeTraits<OLAP_FIELD_TYPE_TINYINT> {
static std::string to_string(const void* src) {
char buf[1024] = {'\0'};
snprintf(buf, sizeof(buf), "%d", *reinterpret_cast<const int8_t*>(src));
return std::string(buf);
}
static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR) {
return convert_int_from_varchar<CppType>(dest, src);
return arithmetic_convert_from_varchar<CppType>(dest, src);
} else if (src_type->type() == OLAP_FIELD_TYPE_CHAR) {
return numeric_convert_from_char<CppType>(dest, src);
}
return OLAPStatus::OLAP_ERR_INVALID_SCHEMA;
}
};

template<>
struct FieldTypeTraits<OLAP_FIELD_TYPE_SMALLINT> : public BaseFieldtypeTraits<OLAP_FIELD_TYPE_SMALLINT> {
static std::string to_string(const void* src) {
char buf[1024] = {'\0'};
snprintf(buf, sizeof(buf), "%d", *reinterpret_cast<const int16_t*>(src));
return std::string(buf);
}
static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR) {
return convert_int_from_varchar<CppType>(dest, src);
}
return OLAPStatus::OLAP_ERR_INVALID_SCHEMA;
}
};
template <FieldType fieldType>
struct NumericFieldtypeTraits<fieldType, false> : public BaseFieldtypeTraits<fieldType> {};

template<>
struct FieldTypeTraits<OLAP_FIELD_TYPE_INT> : public BaseFieldtypeTraits<OLAP_FIELD_TYPE_INT> {
static std::string to_string(const void* src) {
char buf[1024] = {'\0'};
snprintf(buf, sizeof(buf), "%d", *reinterpret_cast<const int32_t *>(src));
return std::string(buf);
}
static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR) {
return convert_int_from_varchar<CppType>(dest, src);
}
return OLAPStatus::OLAP_ERR_INVALID_SCHEMA;
}
};
template <FieldType fieldType>
struct FieldTypeTraits : public NumericFieldtypeTraits<fieldType,
std::is_arithmetic<typename BaseFieldtypeTraits<fieldType>::CppType>::value && std::is_signed<typename BaseFieldtypeTraits<fieldType>::CppType>::value> {};
Copy link
Contributor

@chaoyli chaoyli Nov 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

use std::is_signed for what?

Copy link
Contributor Author

@HappenLee HappenLee Nov 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now， ArithmeticField just to keep the small with old code.
old code only Partial Template Specialization the OLAP_FIELD_TYPE_INT but not OLAP_FIELD_TYPE_UNSIGNED_INT. There is no need to Partial Template Specialization for unsign type, so there check the CppType whether is_signed


template<>
struct FieldTypeTraits<OLAP_FIELD_TYPE_BIGINT> : public BaseFieldtypeTraits<OLAP_FIELD_TYPE_BIGINT> {
struct FieldTypeTraits<OLAP_FIELD_TYPE_BOOL> : public BaseFieldtypeTraits<OLAP_FIELD_TYPE_BOOL> {
static std::string to_string(const void* src) {
char buf[1024] = {'\0'};
snprintf(buf, sizeof(buf), "%ld", *reinterpret_cast<const int64_t*>(src));
snprintf(buf, sizeof(buf), "%d", *reinterpret_cast<const bool*>(src));
return std::string(buf);
}
static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR) {
return convert_int_from_varchar<CppType>(dest, src);
}
return OLAPStatus::OLAP_ERR_INVALID_SCHEMA;
static void set_to_max(void* buf) {
(*(bool*)buf) = true;
}
static void set_to_min(void* buf) {
(*(bool*)buf) = false;
}
};

template<>
struct FieldTypeTraits<OLAP_FIELD_TYPE_LARGEINT> : public BaseFieldtypeTraits<OLAP_FIELD_TYPE_LARGEINT> {
struct FieldTypeTraits<OLAP_FIELD_TYPE_LARGEINT> : public NumericFieldtypeTraits<OLAP_FIELD_TYPE_LARGEINT, true> {
static OLAPStatus from_string(void* buf, const std::string& scan_key) {
int128_t value = 0;

Expand Down Expand Up @@ -699,16 +677,10 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_LARGEINT> : public BaseFieldtypeTraits<OL
static void set_to_min(void* buf) {
*reinterpret_cast<PackedInt128*>(buf) = (int128_t)(1) << 127;
}
static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR) {
return convert_int_from_varchar<CppType>(dest, src);
}
return OLAPStatus::OLAP_ERR_INVALID_SCHEMA;
}
};

template<>
struct FieldTypeTraits<OLAP_FIELD_TYPE_FLOAT> : public BaseFieldtypeTraits<OLAP_FIELD_TYPE_FLOAT> {
struct FieldTypeTraits<OLAP_FIELD_TYPE_FLOAT> : public NumericFieldtypeTraits<OLAP_FIELD_TYPE_FLOAT, true> {
static OLAPStatus from_string(void* buf, const std::string& scan_key) {
CppType value = 0.0f;
if (scan_key.length() > 0) {
Expand All @@ -723,16 +695,10 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_FLOAT> : public BaseFieldtypeTraits<OLAP_
DCHECK(length >= 0) << "gcvt float failed, float value=" << *reinterpret_cast<const CppType *>(src);
return std::string(buf);
}
static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR) {
return convert_float_from_varchar<CppType>(dest, src);
}
return OLAPStatus::OLAP_ERR_INVALID_SCHEMA;
}
};

template<>
struct FieldTypeTraits<OLAP_FIELD_TYPE_DOUBLE> : public BaseFieldtypeTraits<OLAP_FIELD_TYPE_DOUBLE> {
struct FieldTypeTraits<OLAP_FIELD_TYPE_DOUBLE> : public NumericFieldtypeTraits<OLAP_FIELD_TYPE_DOUBLE, true> {
static OLAPStatus from_string(void* buf, const std::string& scan_key) {
CppType value = 0.0;
if (scan_key.length() > 0) {
Expand Down Expand Up @@ -767,10 +733,8 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_DOUBLE> : public BaseFieldtypeTraits<OLAP
*reinterpret_cast<CppType*>(dest) = strtod(buf,&tg);
return OLAPStatus::OLAP_SUCCESS;
}
if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR) {
return convert_float_from_varchar<CppType>(dest, src);
}
return OLAPStatus::OLAP_ERR_INVALID_SCHEMA;

return NumericFieldtypeTraits<OLAP_FIELD_TYPE_DOUBLE, true>::convert_from(dest, src, src_type, mem_pool);
}
};

Expand Down Expand Up @@ -844,7 +808,10 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_DATE> : public BaseFieldtypeTraits<OLAP_F
return OLAPStatus::OLAP_SUCCESS;
}

if (src_type->type() == FieldType::OLAP_FIELD_TYPE_VARCHAR) {
if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR || src_type->type() == OLAP_FIELD_TYPE_CHAR) {
if (src_type->type() == OLAP_FIELD_TYPE_CHAR) {
prepare_char_before_convert(src);
}
using SrcType = typename CppTypeTraits<OLAP_FIELD_TYPE_VARCHAR>::CppType;
auto src_value = *reinterpret_cast<const SrcType*>(src);
DateTimeValue dt;
Expand Down Expand Up @@ -1032,6 +999,9 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_VARCHAR> : public FieldTypeTraits<OLAP_FI
memcpy(slice->data, result.c_str(), result.size());
slice->size = result.size();
return OLAP_SUCCESS;
} else if (src_type->type() == OLAP_FIELD_TYPE_CHAR) {
prepare_char_before_convert(src);
deep_copy(dest, src, mem_pool);
}
return OLAP_ERR_INVALID_SCHEMA;
}
Expand Down
6 changes: 3 additions & 3 deletions docs/en/sql-reference/sql-statements/Data Definition/ALTER TABLE.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,11 +153,11 @@ under the License.
3) Only the type of the column can be modified. The other attributes of the column remain as they are (ie other attributes need to be explicitly written in the statement according to the original attribute, see example 8)
4) The partition column cannot be modified
5) The following types of conversions are currently supported (accuracy loss is guaranteed by the user)
TINYINT/SMALLINT/INT/BIGINT is converted to TINYINT/SMALLINT/INT/BIGINT/DOUBLE.
TINYINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE convert to a wider range of numeric types
TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL is converted to VARCHAR
VARCHAR supports modification of maximum length
Convert VARCHAR to TINYINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE.
Convert VARCHAR to DATE (currently support six formats: "%Y-%m-%d", "%y-%m-%d", "%Y%m%d", "%y%m%d", "%Y/%m/%d, "%y/%m/%d")
Convert VARCHAR/CHAR to TINYINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE.
Convert VARCHAR/CHAR to DATE (currently support six formats: "%Y-%m-%d", "%y-%m-%d", "%Y%m%d", "%y%m%d", "%Y/%m/%d, "%y/%m/%d")
Convert DATETIME to DATE(Only year-month-day information is retained, For example: `2019-12-09 21:47:05` <--> `2019-12-09`)
Convert DATE to DATETIME(Set hour, minute, second to zero, For example: `2019-12-09` <--> `2019-12-09 00:00:00`)
Convert FLOAT to DOUBLE
Expand Down
Loading