diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp index 59bbf18b7f4c1a..2d63a3cfe21f0e 100644 --- a/be/src/olap/base_compaction.cpp +++ b/be/src/olap/base_compaction.cpp @@ -348,8 +348,7 @@ OLAPStatus BaseCompaction::_do_base_compaction(VersionHash new_base_version_hash uint64_t merged_rows = 0; uint64_t filted_rows = 0; OLAPStatus res = OLAP_SUCCESS; - if (_table->data_file_type() == OLAP_DATA_FILE - || _table->data_file_type() == COLUMN_ORIENTED_FILE) { + if (_table->data_file_type() == COLUMN_ORIENTED_FILE) { _table->obtain_header_rdlock(); _table->release_header_lock(); diff --git a/be/src/olap/data_writer.cpp b/be/src/olap/data_writer.cpp index a2a5ebca7d45ef..257c6be3f2a6d9 100644 --- a/be/src/olap/data_writer.cpp +++ b/be/src/olap/data_writer.cpp @@ -51,7 +51,7 @@ ColumnDataWriter::ColumnDataWriter(OLAPTablePtr table, Rowset* index, bool is_pu _segment_writer(NULL), _num_rows(0), _block_id(0), - _max_segment_size(OLAP_MAX_SEGMENT_FILE_SIZE), + _max_segment_size(OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE), _segment(0), _all_num_rows(0), _new_segment_created(false) diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h index c471dde1738e6e..24c1ec6289e4d1 100644 --- a/be/src/olap/olap_define.h +++ b/be/src/olap/olap_define.h @@ -34,8 +34,6 @@ static const uint32_t OLAP_MAX_PATH_LEN = 512; static const uint32_t OLAP_DEFAULT_MAX_PACKED_ROW_BLOCK_SIZE = 1024 * 1024 * 20; // 每个row block压缩前的最大长度,也就是buf的最大长度 static const uint32_t OLAP_DEFAULT_MAX_UNPACKED_ROW_BLOCK_SIZE = 1024 * 1024 * 100; -// 块大小使用uint32_t保存, 这里定义为2G, -static const uint32_t OLAP_MAX_SEGMENT_FILE_SIZE = 2147483648; // 列存储文件的块大小,由于可能会被全部载入内存,所以需要严格控制大小, 这里定义为256MB static const uint32_t OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE = 268435456; // 在列存储文件中, 数据分块压缩, 每个块的默认压缩前的大小 diff --git a/be/src/olap/olap_engine.cpp b/be/src/olap/olap_engine.cpp index a513ec06072173..bb4f8afaffa7ed 100644 --- a/be/src/olap/olap_engine.cpp +++ b/be/src/olap/olap_engine.cpp @@ -2009,15 +2009,11 @@ OLAPStatus OLAPEngine::_create_new_table_header( } else { header->set_keys_type(KeysType::AGG_KEYS); } - if (request.tablet_schema.storage_type == TStorageType::COLUMN) { - header->set_data_file_type(COLUMN_ORIENTED_FILE); - header->set_segment_size(OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE); - header->set_num_rows_per_data_block(config::default_num_rows_per_column_file_block); - } else { - header->set_data_file_type(OLAP_DATA_FILE); - header->set_segment_size(OLAP_MAX_SEGMENT_FILE_SIZE); - header->set_num_rows_per_data_block(config::default_num_rows_per_data_block); - } + DCHECK(request.tablet_schema.storage_type == TStorageType::COLUMN); + header->set_data_file_type(COLUMN_ORIENTED_FILE); + header->set_segment_size(OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE); + header->set_num_rows_per_data_block(config::default_num_rows_per_column_file_block); + // set column information uint32_t i = 0; uint32_t key_count = 0; diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 54d35170069699..9be1d1eeabcd73 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1780,7 +1780,7 @@ OLAPStatus SchemaChangeHandler::schema_version_convert( dest_olap_table, rb_changer, memory_limitation * 1024 * 1024 * 1024); - } else if (true == sc_directly || src_olap_table->data_file_type() == OLAP_DATA_FILE) { + } else if (true == sc_directly) { OLAP_LOG_INFO("doing schema change directly."); sc_procedure = new(nothrow) SchemaChangeDirectly( dest_olap_table, rb_changer); @@ -2000,8 +2000,7 @@ OLAPStatus SchemaChangeHandler::_alter_table(SchemaChangeParams* sc_params) { sc_params->new_olap_table, rb_changer, memory_limitation * 1024 * 1024 * 1024); - } else if (true == sc_directly - || sc_params->ref_olap_table->data_file_type() == OLAP_DATA_FILE) { + } else if (true == sc_directly) { OLAP_LOG_INFO("doing schema change directly."); sc_procedure = new(nothrow) SchemaChangeDirectly( sc_params->new_olap_table, rb_changer); diff --git a/be/test/olap/row_block_test.cpp b/be/test/olap/row_block_test.cpp index c56944cb2ebce7..04961dbcd59655 100644 --- a/be/test/olap/row_block_test.cpp +++ b/be/test/olap/row_block_test.cpp @@ -84,7 +84,7 @@ TEST_F(TestRowBlock, init) { RowBlock block(fields); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = OLAP_DATA_FILE; + block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -96,12 +96,12 @@ TEST_F(TestRowBlock, init) { RowBlock block(fields); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = OLAP_DATA_FILE; + block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = false; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); // num_rows * (num_nullbytes + bigint + char + varchar) - ASSERT_EQ(1024 * (8 + 10 + (4 + 20)), block.buf_len()); + ASSERT_EQ(1024 * (3 + 8 + 10 + (4 + 20)), block.buf_len()); } } @@ -143,7 +143,7 @@ TEST_F(TestRowBlock, write_and_read) { RowBlock block(fields); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = OLAP_DATA_FILE; + block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -259,7 +259,7 @@ TEST_F(TestRowBlock, write_and_read_without_nullbyte) { RowBlock block(fields); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = OLAP_DATA_FILE; + block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = false; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -375,7 +375,7 @@ TEST_F(TestRowBlock, compress_failed) { RowBlock block(fields); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = OLAP_DATA_FILE; + block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -452,7 +452,7 @@ TEST_F(TestRowBlock, decompress_failed) { RowBlock block(fields); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = OLAP_DATA_FILE; + block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -552,7 +552,7 @@ TEST_F(TestRowBlock, find_row) { RowBlock block(fields); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = OLAP_DATA_FILE; + block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -714,7 +714,7 @@ TEST_F(TestRowBlock, clear) { RowBlock block(fields); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = OLAP_DATA_FILE; + block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); @@ -764,7 +764,7 @@ TEST_F(TestRowBlock, pos_limit) { RowBlock block(fields); RowBlockInfo block_info; block_info.row_num = 1024; - block_info.data_file_type = OLAP_DATA_FILE; + block_info.data_file_type = COLUMN_ORIENTED_FILE; block_info.null_supported = true; auto res = block.init(block_info); ASSERT_EQ(OLAP_SUCCESS, res); diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index e8674f80794c7b..598f5cf4098ee1 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -96,7 +96,7 @@ message SchemaChangeStatusMessage { } enum DataFileType { - OLAP_DATA_FILE = 0; + OLAP_DATA_FILE = 0; //Deprecated. Only columnar-wise format is supported. COLUMN_ORIENTED_FILE = 1; }