Transform row-oriented table to columnar-oriented table

be/src/exec/olap_scanner.h

@@ -35,7 +35,7 @@
 #include "runtime/vectorized_row_batch.h"
 
 #include "olap/delete_handler.h"
-#include "olap/i_data.h"
+#include "olap/column_data.h"
 #include "olap/olap_cond.h"
 #include "olap/olap_engine.h"
 #include "olap/reader.h"

be/src/olap/CMakeLists.txt

@@ -24,67 +24,63 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/olap")
 add_library(Olap STATIC
     aggregate_func.cpp
     base_compaction.cpp
+    bit_field_reader.cpp
+    bit_field_writer.cpp
+    bloom_filter.hpp
+    bloom_filter_reader.cpp
+    bloom_filter_writer.cpp
+    byte_buffer.cpp
+    column_data.cpp
+    column_reader.cpp
+    column_writer.cpp
     comparison_predicate.cpp
+    compress.cpp
     cumulative_compaction.cpp
-    delta_writer.cpp
+    data_writer.cpp
     delete_handler.cpp
+    delta_writer.cpp
     field.cpp
     field_info.cpp
     file_helper.cpp
+    file_stream.cpp
     hll.cpp
-    i_data.cpp
     in_list_predicate.cpp
+    in_stream.cpp
     lru_cache.cpp
     memtable.cpp
     merger.cpp
     new_status.cpp
     null_predicate.cpp
     olap_cond.cpp
-    olap_data.cpp
     olap_engine.cpp
     olap_header.cpp
+    olap_header_manager.cpp
     olap_index.cpp
+    olap_meta.cpp
     olap_server.cpp
     olap_snapshot.cpp
-    options.cpp
-    store.cpp
     olap_table.cpp
+    options.cpp
+    out_stream.cpp
     push_handler.cpp
     reader.cpp
     row_block.cpp
     row_cursor.cpp
     rowset.cpp
+    run_length_byte_reader.cpp
+    run_length_byte_writer.cpp
+    run_length_integer_reader.cpp
+    run_length_integer_writer.cpp
     schema_change.cpp
+    segment_reader.cpp
+    segment_writer.cpp
+    serialize.cpp
+    store.cpp
+    stream_index_common.cpp
+    stream_index_reader.cpp
+    stream_index_writer.cpp
+    stream_name.cpp
     types.cpp 
     utils.cpp
     wrapper_field.cpp
-    writer.cpp
-    olap_header_manager.cpp
-    olap_meta.cpp
-    column_file/bit_field_reader.cpp
-    column_file/bit_field_writer.cpp
-    column_file/bloom_filter.hpp
-    column_file/bloom_filter_reader.cpp
-    column_file/bloom_filter_writer.cpp
-    column_file/byte_buffer.cpp
-    column_file/column_data.cpp
-    column_file/column_reader.cpp
-    column_file/column_writer.cpp
-    column_file/compress.cpp
-    column_file/data_writer.cpp
-    column_file/file_stream.cpp
-    column_file/in_stream.cpp
-    column_file/out_stream.cpp
-    column_file/run_length_byte_reader.cpp
-    column_file/run_length_byte_writer.cpp
-    column_file/run_length_integer_reader.cpp
-    column_file/run_length_integer_writer.cpp
-    column_file/segment_reader.cpp
-    column_file/segment_writer.cpp
-    column_file/serialize.cpp
-    column_file/stream_index_common.cpp
-    column_file/stream_index_reader.cpp
-    column_file/stream_index_writer.cpp
-    column_file/stream_name.cpp
 )
-

be/src/olap/base_compaction.cpp

@@ -25,7 +25,7 @@
 
 #include "olap/delete_handler.h"
 #include "olap/merger.h"
-#include "olap/olap_data.h"
+#include "olap/column_data.h"
 #include "olap/olap_engine.h"
 #include "olap/olap_header.h"
 #include "olap/rowset.h"
@@ -105,7 +105,7 @@ OLAPStatus BaseCompaction::run() {
     OLAP_LOG_TRACE("new_base_version_hash", "%ld", new_base_version_hash);
 
     // 2. 获取生成新base需要的data sources
-    vector<IData*> base_data_sources;
+    vector<ColumnData*> base_data_sources;
     _table->acquire_data_sources_by_versions(_need_merged_versions, &base_data_sources);
     if (base_data_sources.empty()) {
         OLAP_LOG_WARNING("fail to acquire need data sources. [table=%s; version=%d]",
@@ -118,7 +118,7 @@ OLAPStatus BaseCompaction::run() {
     {
         DorisMetrics::base_compaction_deltas_total.increment(_need_merged_versions.size());
         int64_t merge_bytes = 0;
-        for (IData* i_data : base_data_sources) {
+        for (ColumnData* i_data : base_data_sources) {
             merge_bytes += i_data->olap_index()->data_size();
         }
         DorisMetrics::base_compaction_bytes_total.increment(merge_bytes);
@@ -133,7 +133,7 @@ OLAPStatus BaseCompaction::run() {
     res = _do_base_compaction(new_base_version_hash,
                              &base_data_sources,
                              &row_count);
-    // 释放不再使用的IData对象
+    // 释放不再使用的ColumnData对象
     _table->release_data_sources(&base_data_sources);
     if (res != OLAP_SUCCESS) {
         OLAP_LOG_WARNING("fail to do base version. [table=%s; version=%d]",
@@ -320,7 +320,7 @@ bool BaseCompaction::_check_whether_satisfy_policy(bool is_manual_trigger,
 }
 
 OLAPStatus BaseCompaction::_do_base_compaction(VersionHash new_base_version_hash,
-                                               vector<IData*>* base_data_sources,
+                                               vector<ColumnData*>* base_data_sources,
                                                uint64_t* row_count) {
     // 1. 生成新base文件对应的olap index
     Rowset* new_base = new (std::nothrow) Rowset(_table.get(),
@@ -397,7 +397,7 @@ OLAPStatus BaseCompaction::_do_base_compaction(VersionHash new_base_version_hash
 
     // Check row num changes
     uint64_t source_rows = 0;
-    for (IData* i_data : *base_data_sources) {
+    for (ColumnData* i_data : *base_data_sources) {
         source_rows += i_data->olap_index()->num_rows();
     }
     bool row_nums_check = config::row_nums_check;
@@ -530,7 +530,7 @@ OLAPStatus BaseCompaction::_validate_delete_file_action() {
     ReadLock rdlock(_table->get_header_lock_ptr());
     const PDelta* lastest_version = _table->lastest_version();
     Version test_version = Version(0, lastest_version->end_version());
-    vector<IData*> test_sources;
+    vector<ColumnData*> test_sources;
     _table->acquire_data_sources(test_version, &test_sources);
 
     if (test_sources.size() == 0) {

be/src/olap/base_compaction.h

@@ -27,7 +27,7 @@
 
 namespace doris {
 
-class IData;
+class ColumnData;
 class Rowset;
 
 // @brief 实现对START_BASE_COMPACTION命令的处理逻辑，并返回处理结果
@@ -82,14 +82,14 @@ class BaseCompaction {
     // 
     // 输入参数：
     // - new_base_version_hash: 新Base的VersionHash
-    // - base_data_sources: 生成新Base需要的IData*
+    // - base_data_sources: 生成新Base需要的ColumnData*
     // - row_count: 生成Base过程中产生的row_count
     //
     // 返回值：
     // - 如果执行成功，则返回OLAP_SUCCESS；
     // - 其它情况下，返回相应的错误码
     OLAPStatus _do_base_compaction(VersionHash new_base_version_hash,
-                                  std::vector<IData*>* base_data_sources,
+                                  std::vector<ColumnData*>* base_data_sources,
                                   uint64_t* row_count);
 
     // 更新Header使得修改对外可见

be/src/olap/column_file/bit_field_reader.cpp → be/src/olap/bit_field_reader.cpp

@@ -15,14 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "olap/column_file/bit_field_reader.h"
+#include "olap/bit_field_reader.h"
 
-#include "olap/column_file/column_reader.h"
-#include "olap/column_file/in_stream.h"
-#include "olap/column_file/run_length_byte_reader.h"
+#include "olap/column_reader.h"
+#include "olap/in_stream.h"
+#include "olap/run_length_byte_reader.h"
 
 namespace doris {
-namespace column_file {
 
 BitFieldReader::BitFieldReader(ReadOnlyFileStream* input) : 
         _input(input),
@@ -128,6 +127,5 @@ OLAPStatus BitFieldReader::skip(uint64_t num_values) {
     return OLAP_SUCCESS;
 }
 
-}  // namespace column_file
 }  // namespace doris
 

be/src/olap/column_file/bit_field_reader.h → be/src/olap/bit_field_reader.h

@@ -18,11 +18,10 @@
 #ifndef DORIS_BE_SRC_OLAP_COLUMN_FILE_BIT_FIELD_READER_H
 #define DORIS_BE_SRC_OLAP_COLUMN_FILE_BIT_FIELD_READER_H
 
-#include "olap/column_file/stream_index_reader.h"
+#include "olap/stream_index_reader.h"
 #include "olap/olap_define.h"
 
 namespace doris {
-namespace column_file {
 
 class ReadOnlyFileStream;
 class RunLengthByteReader;
@@ -49,7 +48,6 @@ class BitFieldReader {
     DISALLOW_COPY_AND_ASSIGN(BitFieldReader);
 };
 
-}  // namespace column_file
 }  // namespace doris
 
 #endif // DORIS_BE_SRC_OLAP_COLUMN_FILE_BIT_FIELD_READER_H

be/src/olap/column_file/bit_field_writer.cpp → be/src/olap/bit_field_writer.cpp

@@ -17,10 +17,9 @@
 
 #include "bit_field_writer.h"
 #include <gen_cpp/column_data_file.pb.h>
-#include "olap/column_file/run_length_byte_writer.h"
+#include "olap/run_length_byte_writer.h"
 
 namespace doris {
-namespace column_file {
 
 BitFieldWriter::BitFieldWriter(OutStream* output) : 
         _output(output),
@@ -104,5 +103,4 @@ void BitFieldWriter::get_position(PositionEntryWriter* index_entry) const {
     //                "recorded position count: %d", index_entry->positions_size());
 }
 
-}  // namespace column_file
 }  // namespace doris

be/src/olap/column_file/bit_field_writer.h → be/src/olap/bit_field_writer.h

@@ -18,11 +18,10 @@
 #ifndef DORIS_BE_SRC_OLAP_COLUMN_FILE_BIT_FIELD_WRITER_H
 #define DORIS_BE_SRC_OLAP_COLUMN_FILE_BIT_FIELD_WRITER_H
 
-#include "olap/column_file/stream_index_writer.h"
+#include "olap/stream_index_writer.h"
 #include "olap/olap_define.h"
 
 namespace doris {
-namespace column_file {
 
 class OutStream;
 class RunLengthByteWriter;
@@ -47,7 +46,6 @@ class BitFieldWriter {
     DISALLOW_COPY_AND_ASSIGN(BitFieldWriter);
 };
 
-}  // namespace column_file
 }  // namespace doris
 
 #endif // DORIS_BE_SRC_OLAP_COLUMN_FILE_BIT_FIELD_WRITER_H

be/src/olap/column_file/bloom_filter.hpp → be/src/olap/bloom_filter.hpp

@@ -28,7 +28,6 @@
 #include "util/hash_util.hpp"
 
 namespace doris {
-namespace column_file {
 
 static const uint64_t DEFAULT_SEED = 104729;
 static const uint64_t BLOOM_FILTER_NULL_HASHCODE = 2862933555777941757ULL;
@@ -297,7 +296,6 @@ class BloomFilter {
     uint32_t _hash_function_num;
 };
 
-}  // namespace column_file
 }  // namespace doris
 
 #endif // DORIS_BE_SRC_OLAP_COLUMN_FILE_BLOOM_FILTER_HPP

be/src/olap/column_file/bloom_filter_reader.cpp → be/src/olap/bloom_filter_reader.cpp

@@ -15,10 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "olap/column_file/bloom_filter_reader.h"
+#include "olap/bloom_filter_reader.h"
 
 namespace doris {
-namespace column_file {
 
 BloomFilterIndexReader::~BloomFilterIndexReader() {
     _entry.reset();
@@ -65,5 +64,4 @@ size_t BloomFilterIndexReader::entry_count() {
     return _entry_count;
 }
 
-}  // namespace column_file
 }  // namespace doris

be/src/olap/column_file/bloom_filter_reader.h → be/src/olap/bloom_filter_reader.h

@@ -20,11 +20,10 @@
 
 #include <vector>
 
-#include "olap/column_file/bloom_filter.hpp"
-#include "olap/column_file/bloom_filter_writer.h"
+#include "olap/bloom_filter.hpp"
+#include "olap/bloom_filter_writer.h"
 
 namespace doris {
-namespace column_file {
 
 // Each bloom filter index contains mutiple bloom filter entries,
 //     each of which is related to a data block.
@@ -70,6 +69,5 @@ class BloomFilterIndexReader {
     BloomFilter _entry;
 };
 
-} // namespace column_file
 } // namespace doris
 #endif // DORIS_BE_SRC_OLAP_COLUMN_FILE_BLOOM_FILTER_READER_H

be/src/olap/column_file/bloom_filter_writer.cpp → be/src/olap/bloom_filter_writer.cpp

@@ -15,12 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "olap/column_file/bloom_filter_writer.h"
+#include "olap/bloom_filter_writer.h"
 
 #include <vector>
 
 namespace doris {
-namespace column_file {
 
 BloomFilterIndexWriter::~BloomFilterIndexWriter() {
     for (std::vector<BloomFilter*>::iterator it = _bloom_filters.begin();
@@ -107,5 +106,4 @@ OLAPStatus BloomFilterIndexWriter::write_to_buffer(char* buffer, size_t buffer_s
     return res;
 }
 
-} // namespace column_file
 } // namespace doris

be/src/olap/column_file/bloom_filter_writer.h → be/src/olap/bloom_filter_writer.h

@@ -20,11 +20,10 @@
 
 #include <vector>
 
-#include "olap/column_file/bloom_filter.hpp"
-#include "olap/column_file/out_stream.h"
+#include "olap/bloom_filter.hpp"
+#include "olap/out_stream.h"
 
 namespace doris {
-namespace column_file {
 
 class BloomFilterIndexWriter {
 public:
@@ -41,6 +40,5 @@ class BloomFilterIndexWriter {
     BloomFilterIndexHeader _header;
 };
 
-} // namespace column_file
 } // namespace doris
 #endif // DORIS_BE_SRC_OLAP_COLUMN_FILE_BLOOM_FILTER_WRITER_H

be/src/olap/column_file/byte_buffer.cpp → be/src/olap/byte_buffer.cpp

@@ -20,7 +20,6 @@
 #include "olap/utils.h"
 
 namespace doris {
-namespace column_file {
 
 ByteBuffer::ByteBuffer() : 
         _array(NULL),
@@ -194,5 +193,4 @@ OLAPStatus ByteBuffer::put(const char* src, uint64_t src_size, uint64_t offset,
     return OLAP_SUCCESS;
 }
 
-}  // namespace column_file
 }  // namespace doris