diff --git a/be/src/olap/field.h b/be/src/olap/field.h index 7a2c2a43ec8175..ad513778b736d0 100644 --- a/be/src/olap/field.h +++ b/be/src/olap/field.h @@ -55,9 +55,9 @@ class Field { inline size_t field_size() const { return size() + 1; } inline size_t index_size() const { return _index_size; } - inline void set_to_max(char* buf) const { return _type_info->set_to_max(buf); } + virtual inline void set_to_max(char* buf) const { return _type_info->set_to_max(buf); } inline void set_to_min(char* buf) const { return _type_info->set_to_min(buf); } - inline char* allocate_value_from_arena(Arena* arena) const { return _type_info->allocate_value_from_arena(arena); } + virtual inline char* allocate_value_from_arena(Arena* arena) const { return arena->Allocate(_type_info->size()); } inline void agg_update(RowCursorCell* dest, const RowCursorCell& src, MemPool* mem_pool = nullptr) const { _agg_info->update(dest, src, mem_pool); @@ -200,10 +200,6 @@ class Field { _type_info->deep_copy_with_arena(dest, src, arena); } - inline void direct_copy_content(char* dest, const char* src) const { - _type_info->direct_copy(dest, src); - } - // Copy srouce content to destination in index format. template void to_index(DstCellType* dst, const SrcCellType& src) const; @@ -259,6 +255,14 @@ class Field { // 长度,单位为字节 // 除字符串外,其它类型都是确定的 uint32_t _length; + + char* allocate_string_value_from_arena(Arena* arena) const { + char* type_value = arena->Allocate(sizeof(Slice)); + auto slice = reinterpret_cast(type_value); + slice->size = _length; + slice->data = arena->Allocate(slice->size); + return type_value; + }; }; template @@ -378,6 +382,16 @@ class CharField: public Field { CharField* clone() const override { return new CharField(*this); } + + char* allocate_value_from_arena(Arena* arena) const override { + return Field::allocate_string_value_from_arena(arena); + } + + void set_to_max(char* ch) const override { + auto slice = reinterpret_cast(ch); + slice->size = _length; + memset(slice->data, 0xFF, slice->size); + } }; class VarcharField: public Field { @@ -389,6 +403,7 @@ class VarcharField: public Field { return _length - OLAP_STRING_MAX_BYTES; } + // minus OLAP_STRING_MAX_BYTES here just for being compatible with old storage format char* allocate_memory(char* cell_ptr, char* variable_ptr) const override { auto slice = (Slice*)cell_ptr; slice->data = variable_ptr; @@ -400,6 +415,16 @@ class VarcharField: public Field { VarcharField* clone() const override { return new VarcharField(*this); } + + char* allocate_value_from_arena(Arena* arena) const override { + return Field::allocate_string_value_from_arena(arena); + } + + void set_to_max(char* ch) const override { + auto slice = reinterpret_cast(ch); + slice->size = _length - OLAP_STRING_MAX_BYTES; + memset(slice->data, 0xFF, slice->size); + } }; class BitmapAggField: public Field { diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h index 685a9ef1b8ef1c..f4b40f1c07ad6f 100644 --- a/be/src/olap/olap_define.h +++ b/be/src/olap/olap_define.h @@ -53,9 +53,6 @@ static constexpr uint32_t OLAP_COMPACTION_DEFAULT_CANDIDATE_SIZE = 10; // the max length supported for varchar type static const uint16_t OLAP_STRING_MAX_LENGTH = 65535; -//the max length supported for char type -static const uint16_t OLAP_CHAR_MAX_LENGTH = 255; - static const int32_t PREFERRED_SNAPSHOT_VERSION = 3; // the max bytes for stored string length diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp index 4faf37270ae734..d9ea6f635dc4ec 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.cpp +++ b/be/src/olap/rowset/segment_v2/column_writer.cpp @@ -72,13 +72,13 @@ class NullBitmapBuilder { }; ColumnWriter::ColumnWriter(const ColumnWriterOptions& opts, - const TypeInfo* typeinfo, + std::unique_ptr field, bool is_nullable, WritableFile* output_file) : _opts(opts), - _type_info(typeinfo), _is_nullable(is_nullable), - _output_file(output_file) { + _output_file(output_file), + _field(std::move(field)) { } ColumnWriter::~ColumnWriter() { @@ -92,7 +92,7 @@ ColumnWriter::~ColumnWriter() { } Status ColumnWriter::init() { - RETURN_IF_ERROR(EncodingInfo::get(_type_info, _opts.encoding_type, &_encoding_info)); + RETURN_IF_ERROR(EncodingInfo::get(_field->type_info(), _opts.encoding_type, &_encoding_info)); if (_opts.compression_type != NO_COMPRESSION) { RETURN_IF_ERROR(get_block_compression_codec(_opts.compression_type, &_compress_codec)); } @@ -105,7 +105,7 @@ Status ColumnWriter::init() { if (page_builder == nullptr) { return Status::NotSupported( Substitute("Failed to create page builder for type $0 and encoding $1", - _type_info->type(), _opts.encoding_type)); + _field->type(), _opts.encoding_type)); } _page_builder.reset(page_builder); // create ordinal builder @@ -115,7 +115,7 @@ Status ColumnWriter::init() { _null_bitmap_builder.reset(new NullBitmapBuilder()); } if (_opts.need_zone_map) { - _column_zone_map_builder.reset(new ColumnZoneMapBuilder(_type_info)); + _column_zone_map_builder.reset(new ColumnZoneMapBuilder(_field.get())); } return Status::OK(); } @@ -148,7 +148,7 @@ Status ColumnWriter::_append_data(const uint8_t** ptr, size_t num_rows) { bool is_page_full = (num_written < remaining); remaining -= num_written; _next_rowid += num_written; - *ptr += _type_info->size() * num_written; + *ptr += _field->size() * num_written; // we must write null bits after write data, because we don't // know how many rows can be written into current page if (_is_nullable) { @@ -240,7 +240,7 @@ Status ColumnWriter::write_zone_map() { } void ColumnWriter::write_meta(ColumnMetaPB* meta) { - meta->set_type(_type_info->type()); + meta->set_type(_field->type()); meta->set_encoding(_opts.encoding_type); meta->set_compression(_opts.compression_type); meta->set_is_nullable(_is_nullable); diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h index b881c13a3ecd00..6626b4d881431d 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.h +++ b/be/src/olap/rowset/segment_v2/column_writer.h @@ -57,7 +57,7 @@ class PageBuilder; class ColumnWriter { public: ColumnWriter(const ColumnWriterOptions& opts, - const TypeInfo* typeinfo, + std::unique_ptr field, bool is_nullable, WritableFile* output_file); ~ColumnWriter(); @@ -138,7 +138,6 @@ class ColumnWriter { private: ColumnWriterOptions _opts; - const TypeInfo* _type_info = nullptr; bool _is_nullable; WritableFile* _output_file = nullptr; @@ -154,6 +153,7 @@ class ColumnWriter { std::unique_ptr _null_bitmap_builder; std::unique_ptr _ordinal_index_builder; std::unique_ptr _column_zone_map_builder; + std::unique_ptr _field; PagePointer _ordinal_index_pp; PagePointer _zone_map_pp; diff --git a/be/src/olap/rowset/segment_v2/column_zone_map.cpp b/be/src/olap/rowset/segment_v2/column_zone_map.cpp index 61dd3f25639810..c29896c06f3737 100644 --- a/be/src/olap/rowset/segment_v2/column_zone_map.cpp +++ b/be/src/olap/rowset/segment_v2/column_zone_map.cpp @@ -23,11 +23,10 @@ namespace doris { namespace segment_v2 { -ColumnZoneMapBuilder::ColumnZoneMapBuilder(const TypeInfo* type_info) : _type_info(type_info) { +ColumnZoneMapBuilder::ColumnZoneMapBuilder(Field* field) : _field(field) { PageBuilderOptions options; options.data_page_size = 0; _page_builder.reset(new BinaryPlainPageBuilder(options)); - _field.reset(FieldFactory::create_by_type(_type_info->type())); _zone_map.min_value = _field->allocate_value_from_arena(&_arena); _zone_map.max_value = _field->allocate_value_from_arena(&_arena); @@ -38,12 +37,12 @@ Status ColumnZoneMapBuilder::add(const uint8_t *vals, size_t count) { if (vals != nullptr) { for (int i = 0; i < count; ++i) { if (_field->compare(_zone_map.min_value, (char *)vals) > 0) { - _field->direct_copy_content(_zone_map.min_value, (const char *)vals); + _field->type_info()->direct_copy(_zone_map.min_value, (const char *)vals); } if (_field->compare(_zone_map.max_value, (char *)vals) < 0) { - _field->direct_copy_content(_zone_map.max_value, (const char *)vals); + _field->type_info()->direct_copy(_zone_map.max_value, (const char *)vals); } - vals += _type_info->size(); + vals += _field->size(); if (!_zone_map.has_not_null) { _zone_map.has_not_null = true; } diff --git a/be/src/olap/rowset/segment_v2/column_zone_map.h b/be/src/olap/rowset/segment_v2/column_zone_map.h index 3e2b7ac78f6a12..e103e7b240013f 100644 --- a/be/src/olap/rowset/segment_v2/column_zone_map.h +++ b/be/src/olap/rowset/segment_v2/column_zone_map.h @@ -50,7 +50,7 @@ struct ZoneMap { // The binary is encoded by BinaryPlainPageBuilder class ColumnZoneMapBuilder { public: - ColumnZoneMapBuilder(const TypeInfo* type_info); + ColumnZoneMapBuilder(Field* field); Status add(const uint8_t* vals, size_t count); @@ -68,9 +68,8 @@ class ColumnZoneMapBuilder { void _reset_zone_map(); private: - const TypeInfo* _type_info; std::unique_ptr _page_builder; - std::unique_ptr _field; + Field* _field; // memory will be managed by arena ZoneMap _zone_map; Arena _arena; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index c4afd2a8527052..a5d33070c10746 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -55,17 +55,17 @@ Status SegmentWriter::init(uint32_t write_mbytes_per_sec) { bool is_nullable = column.is_nullable(); column_meta->set_is_nullable(is_nullable); - // TODO(zc): we can add type_info into TabletColumn? - const TypeInfo* type_info = get_type_info(column.type()); - DCHECK(type_info != nullptr); - ColumnWriterOptions opts; opts.compression_type = segment_v2::CompressionTypePB::LZ4F; // now we create zone map for key columns if (column.is_key()) { opts.need_zone_map = true; } - std::unique_ptr writer(new ColumnWriter(opts, type_info, is_nullable, _output_file.get())); + + std::unique_ptr field(FieldFactory::create(column)); + DCHECK(field.get() != nullptr); + + std::unique_ptr writer(new ColumnWriter(opts, std::move(field), is_nullable, _output_file.get())); RETURN_IF_ERROR(writer->init()); _column_writers.push_back(std::move(writer)); } diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp index 36704fcdd9d82d..d7f976851e609f 100644 --- a/be/src/olap/types.cpp +++ b/be/src/olap/types.cpp @@ -19,6 +19,8 @@ namespace doris { +void (*FieldTypeTraits::set_to_max)(void*) = nullptr; + template TypeInfo::TypeInfo(TypeTraitsClass t) : _equal(TypeTraitsClass::equal), @@ -27,7 +29,6 @@ TypeInfo::TypeInfo(TypeTraitsClass t) _deep_copy(TypeTraitsClass::deep_copy), _deep_copy_with_arena(TypeTraitsClass::deep_copy_with_arena), _direct_copy(TypeTraitsClass::direct_copy), - _allocate_value_from_arena(TypeTraitsClass::allocate_value_from_arena), _from_string(TypeTraitsClass::from_string), _to_string(TypeTraitsClass::to_string), _set_to_max(TypeTraitsClass::set_to_max), diff --git a/be/src/olap/types.h b/be/src/olap/types.h index bd381106a02c57..78bc09315cc3a7 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -64,10 +64,6 @@ class TypeInfo { _direct_copy(dest, src); } - inline char* allocate_value_from_arena(Arena* arena) const { - return _allocate_value_from_arena(arena); - } - OLAPStatus from_string(void* buf, const std::string& scan_key) const { return _from_string(buf, scan_key); } @@ -89,7 +85,6 @@ class TypeInfo { void (*_deep_copy)(void* dest, const void* src, MemPool* mem_pool); void (*_deep_copy_with_arena)(void* dest, const void* src, Arena* arena); void (*_direct_copy)(void* dest, const void* src); - char* (*_allocate_value_from_arena)(Arena* arena); OLAPStatus (*_from_string)(void* buf, const std::string& scan_key); std::string (*_to_string)(const void* src); @@ -218,10 +213,6 @@ struct BaseFieldtypeTraits : public CppTypeTraits { return HashUtil::hash(data, sizeof(CppType), seed); } - static inline char* allocate_value_from_arena(Arena* arena) { - return arena->Allocate(sizeof(CppType)); - } - static std::string to_string(const void* src) { std::stringstream stream; stream << *reinterpret_cast(src); @@ -563,12 +554,10 @@ struct FieldTypeTraits : public BaseFieldtypeTraitsdata, r_slice->data, r_slice->size); l_slice->size = r_slice->size; } - static void set_to_max(void* buf) { - // this function is used by scan key, - // the size may be greater than length in schema. - auto slice = reinterpret_cast(buf); - memset(slice->data, 0xff, slice->size); - } + + // using field.set_to_max to set varchar/char,not here + static void (*set_to_max)(void*); + static void set_to_min(void* buf) { auto slice = reinterpret_cast(buf); memset(slice->data, 0, slice->size); @@ -577,13 +566,6 @@ struct FieldTypeTraits : public BaseFieldtypeTraits(data); return HashUtil::hash(slice->data, slice->size, seed); } - static char* allocate_value_from_arena(Arena* arena) { - char* type_value = arena->Allocate(sizeof(Slice)); - auto slice = reinterpret_cast(type_value); - slice->size = OLAP_CHAR_MAX_LENGTH; - slice->data = arena->Allocate(OLAP_CHAR_MAX_LENGTH); - return type_value; - } }; template<> @@ -601,22 +583,11 @@ struct FieldTypeTraits : public FieldTypeTraitssize = value_len; return OLAP_SUCCESS; } - static void set_to_max(void* buf) { - auto slice = reinterpret_cast(buf); - slice->size = 1; - memset(slice->data, 0xFF, 1); - } + static void set_to_min(void* buf) { auto slice = reinterpret_cast(buf); slice->size = 0; } - static char* allocate_value_from_arena(Arena* arena) { - char* type_value = arena->Allocate(sizeof(Slice)); - auto slice = reinterpret_cast(type_value); - slice->size = OLAP_STRING_MAX_LENGTH; - slice->data = arena->Allocate(OLAP_STRING_MAX_LENGTH); - return type_value; - } }; template<> diff --git a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp index 79e82df2462e17..91b82960a7990b 100644 --- a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp +++ b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp @@ -17,6 +17,7 @@ #include "olap/rowset/segment_v2/column_reader.h" #include "olap/rowset/segment_v2/column_writer.h" +#include "olap/tablet_schema_helper.h" #include "olap/decimal12.h" #include @@ -62,7 +63,14 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s writer_opts.compression_type = segment_v2::CompressionTypePB::LZ4F; writer_opts.need_zone_map = true; - ColumnWriter writer(writer_opts, type_info, true, wfile.get()); + TabletColumn column(OLAP_FIELD_AGGREGATION_NONE, type); + if (type == OLAP_FIELD_TYPE_VARCHAR) { + column = create_varchar_key(1); + } else if (type == OLAP_FIELD_TYPE_CHAR) { + column = create_char_key(1); + } + std::unique_ptr field(FieldFactory::create(column)); + ColumnWriter writer(writer_opts, std::move(field), true, wfile.get()); st = writer.init(); ASSERT_TRUE(st.ok()); @@ -113,6 +121,7 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s Arena arena; Type vals[1024]; + Type* vals_ = vals; uint8_t is_null[1024]; ColumnBlock col(type_info, (uint8_t*)vals, is_null, &arena); @@ -126,7 +135,13 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s // << ", src[idx]=" << src[idx] << ", vals[j]=" << vals[j]; ASSERT_EQ(BitmapTest(src_is_null, idx), BitmapTest(is_null, j)); if (!BitmapTest(is_null, j)) { - ASSERT_EQ(src[idx], vals[j]); + if (type == OLAP_FIELD_TYPE_VARCHAR || type == OLAP_FIELD_TYPE_CHAR) { + Slice* src_slice = (Slice*)src_data; + Slice* dst_slice = (Slice*)vals_; + ASSERT_EQ(src_slice[idx].to_string(), dst_slice[j].to_string()) << "j:" << j; + } else { + ASSERT_EQ(src[idx], vals[j]); + } } idx++; } @@ -155,7 +170,13 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s // << ", src[idx]=" << src[idx] << ", vals[j]=" << vals[j]; ASSERT_EQ(BitmapTest(src_is_null, idx), BitmapTest(is_null, j)); if (!BitmapTest(is_null, j)) { - ASSERT_EQ(src[idx], vals[j]); + if (type == OLAP_FIELD_TYPE_VARCHAR || type == OLAP_FIELD_TYPE_CHAR) { + Slice* src_slice = (Slice*)src_data; + Slice* dst_slice = (Slice*)vals; + ASSERT_EQ(src_slice[idx].to_string(), dst_slice[j].to_string()); + } else { + ASSERT_EQ(src[idx], vals[j]); + } } idx++; } @@ -203,6 +224,8 @@ TEST_F(ColumnReaderWriterTest, test_nullable) { } TEST_F(ColumnReaderWriterTest, test_types) { + Arena arena; + size_t num_uint8_rows = 1024 * 1024; uint8_t* is_null = new uint8_t[num_uint8_rows]; @@ -210,13 +233,21 @@ TEST_F(ColumnReaderWriterTest, test_types) { uint24_t* date_vals = new uint24_t[num_uint8_rows]; uint64_t* datetime_vals = new uint64_t[num_uint8_rows]; decimal12_t* decimal_vals = new decimal12_t[num_uint8_rows]; + Slice* varchar_vals = new Slice[num_uint8_rows]; + Slice* char_vals = new Slice[num_uint8_rows]; for (int i = 0; i < num_uint8_rows; ++i) { bool_vals[i] = i % 2; date_vals[i] = i + 33; datetime_vals[i] = i + 33; decimal_vals[i] = decimal12_t(i, i); // 1.000000001 + + set_column_value_by_type(OLAP_FIELD_TYPE_VARCHAR, i, (char*)&varchar_vals[i], &arena); + set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, i, (char*)&char_vals[i], &arena, 8); + BitmapChange(is_null, i, (i % 4) == 0); } + test_nullable_data((uint8_t*)char_vals, is_null, num_uint8_rows, "null_char_bs"); + test_nullable_data((uint8_t*)varchar_vals, is_null, num_uint8_rows, "null_varchar_bs"); test_nullable_data((uint8_t*)bool_vals, is_null, num_uint8_rows, "null_bool_bs"); test_nullable_data((uint8_t*)date_vals, is_null, num_uint8_rows / 3, "null_date_bs"); @@ -230,6 +261,8 @@ TEST_F(ColumnReaderWriterTest, test_types) { } test_nullable_data((uint8_t*)decimal_vals, is_null, num_uint8_rows / 12, "null_decimal_bs"); + delete[] char_vals; + delete[] varchar_vals; delete[] is_null; delete[] bool_vals; delete[] date_vals; diff --git a/be/test/olap/rowset/segment_v2/column_zone_map_test.cpp b/be/test/olap/rowset/segment_v2/column_zone_map_test.cpp index b30157fe4228a0..98c3a3ccef10bc 100644 --- a/be/test/olap/rowset/segment_v2/column_zone_map_test.cpp +++ b/be/test/olap/rowset/segment_v2/column_zone_map_test.cpp @@ -19,15 +19,15 @@ #include #include "olap/rowset/segment_v2/column_zone_map.h" +#include "olap/tablet_schema_helper.h" namespace doris { namespace segment_v2 { class ColumnZoneMapTest : public testing::Test { public: - void test_string(FieldType type) { - TypeInfo *type_info = get_type_info(OLAP_FIELD_TYPE_CHAR); - ColumnZoneMapBuilder builder(type_info); + void test_string(Field* field) { + ColumnZoneMapBuilder builder(field); std::vector values1 = {"aaaa", "bbbb", "cccc", "dddd", "eeee", "ffff"}; for (auto value : values1) { builder.add((const uint8_t*)&value, 1); @@ -67,8 +67,10 @@ class ColumnZoneMapTest : public testing::Test { // Test for int TEST_F(ColumnZoneMapTest, NormalTestIntPage) { - TypeInfo* type_info = get_type_info(OLAP_FIELD_TYPE_INT); - ColumnZoneMapBuilder builder(type_info); + TabletColumn int_column = create_int_key(0); + Field* field = FieldFactory::create(int_column); + + ColumnZoneMapBuilder builder(field); std::vector values1 = {1, 10, 11, 20, 21, 22}; for (auto value : values1) { builder.add((const uint8_t*)&value, 1); @@ -108,12 +110,16 @@ TEST_F(ColumnZoneMapTest, NormalTestIntPage) { // Test for string TEST_F(ColumnZoneMapTest, NormalTestVarcharPage) { - test_string(OLAP_FIELD_TYPE_VARCHAR); + TabletColumn varchar_column = create_varchar_key(0); + Field* field = FieldFactory::create(varchar_column); + test_string(field); } // Test for string TEST_F(ColumnZoneMapTest, NormalTestCharPage) { - test_string(OLAP_FIELD_TYPE_CHAR); + TabletColumn char_column = create_char_key(0); + Field* field = FieldFactory::create(char_column); + test_string(field); } } diff --git a/be/test/olap/rowset/segment_v2/segment_test.cpp b/be/test/olap/rowset/segment_v2/segment_test.cpp index 01121ce872361c..1e435b247fcd8a 100644 --- a/be/test/olap/rowset/segment_v2/segment_test.cpp +++ b/be/test/olap/rowset/segment_v2/segment_test.cpp @@ -606,29 +606,6 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) { } } -void set_column_value_by_type(FieldType fieldType, int src, char* target, Arena* _arena, size_t _length = 0) { - if (fieldType == OLAP_FIELD_TYPE_CHAR) { - char* src_value = &std::to_string(src)[0]; - int src_len = strlen(src_value); - - auto* dest_slice = (Slice*)target; - dest_slice->size = _length; - dest_slice->data = _arena->Allocate(dest_slice->size); - memcpy(dest_slice->data, src_value, src_len); - memset(dest_slice->data + src_len, 0, dest_slice->size - src_len); - } else if (fieldType == OLAP_FIELD_TYPE_VARCHAR) { - char* src_value = &std::to_string(src)[0]; - int src_len = strlen(src_value); - - auto* dest_slice = (Slice*)target; - dest_slice->size = src_len; - dest_slice->data = _arena->Allocate(src_len); - std::memcpy(dest_slice->data, src_value, src_len); - } else { - *(int*)target = src; - } -} - TEST_F(SegmentReaderWriterTest, TestStringDict) { size_t num_rows_per_block = 10; Arena _arena; diff --git a/be/test/olap/storage_types_test.cpp b/be/test/olap/storage_types_test.cpp index 745b087c876e64..8a11dd5dd7e937 100644 --- a/be/test/olap/storage_types_test.cpp +++ b/be/test/olap/storage_types_test.cpp @@ -22,6 +22,7 @@ #include "runtime/mem_tracker.h" #include "runtime/mem_pool.h" #include "util/slice.h" +#include "olap/field.h" namespace doris { @@ -70,10 +71,13 @@ void common_test(typename TypeTraits::CppType src_val) { } } -void test_char(FieldType field_type, Slice src_val) { - TypeInfo* type = get_type_info(field_type); +template +void test_char(Slice src_val) { + Field* field = FieldFactory::create_by_type(fieldType); + field->_length = src_val.size; + const TypeInfo* type = field->type_info(); - ASSERT_EQ(field_type, type->type()); + ASSERT_EQ(field->type(), fieldType); ASSERT_EQ(sizeof(src_val), type->size()); { char buf[64]; @@ -95,7 +99,7 @@ void test_char(FieldType field_type, Slice src_val) { { char buf[64]; Slice dst_val(buf, sizeof(buf)); - type->set_to_min((char*)&dst_val); + field->set_to_min((char*)&dst_val); ASSERT_FALSE(type->equal((char*)&src_val, (char*)&dst_val)); ASSERT_TRUE(type->cmp((char*)&src_val, (char*)&dst_val) > 0); @@ -104,7 +108,7 @@ void test_char(FieldType field_type, Slice src_val) { { char buf[64]; Slice dst_val(buf, sizeof(buf)); - type->set_to_max((char*)&dst_val); + field->set_to_max((char*)&dst_val); ASSERT_FALSE(type->equal((char*)&src_val, (char*)&dst_val)); ASSERT_TRUE(type->cmp((char*)&src_val, (char*)&dst_val) < 0); @@ -113,12 +117,12 @@ void test_char(FieldType field_type, Slice src_val) { template<> void common_test(Slice src_val) { - test_char(OLAP_FIELD_TYPE_CHAR, src_val); + test_char(src_val); } template<> void common_test(Slice src_val) { - test_char(OLAP_FIELD_TYPE_VARCHAR, src_val); + test_char(src_val); } TEST(TypesTest, copy_and_equal) { diff --git a/be/test/olap/tablet_schema_helper.h b/be/test/olap/tablet_schema_helper.h index 73ecdf323e3ff0..7e7082f7d2aedd 100644 --- a/be/test/olap/tablet_schema_helper.h +++ b/be/test/olap/tablet_schema_helper.h @@ -74,9 +74,32 @@ TabletColumn create_varchar_key(int32_t id, bool is_nullable = true) { column._type = OLAP_FIELD_TYPE_VARCHAR; column._is_key = true; column._is_nullable = is_nullable; - column._length = 4; + column._length = 8; column._index_length = 4; return column; } +void set_column_value_by_type(FieldType fieldType, int src, char* target, Arena* _arena, size_t _length = 0) { + if (fieldType == OLAP_FIELD_TYPE_CHAR) { + char* src_value = &std::to_string(src)[0]; + int src_len = strlen(src_value); + + auto* dest_slice = (Slice*)target; + dest_slice->size = _length; + dest_slice->data = _arena->Allocate(dest_slice->size); + memcpy(dest_slice->data, src_value, src_len); + memset(dest_slice->data + src_len, 0, dest_slice->size - src_len); + } else if (fieldType == OLAP_FIELD_TYPE_VARCHAR) { + char* src_value = &std::to_string(src)[0]; + int src_len = strlen(src_value); + + auto* dest_slice = (Slice*)target; + dest_slice->size = src_len; + dest_slice->data = _arena->Allocate(src_len); + std::memcpy(dest_slice->data, src_value, src_len); + } else { + *(int*)target = src; + } +} + }