diff --git a/be/src/olap/rowset/dfile/column_reader.h b/be/src/olap/rowset/dfile/column_reader.h new file mode 100644 index 00000000000000..baaf31a9b7a422 --- /dev/null +++ b/be/src/olap/rowset/dfile/column_reader.h @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_ROWSET_DFILE_COLUMN_READER_H +#define DORIS_BE_SRC_OLAP_ROWSET_DFILE_COLUMN_READER_H + +#include "runtime/vectorized_row_batch.h" + +namespace doris { + +namespace dfile { + +class ColumnReader { +public: + ColumnReader() { } + + bool init(); + + // Seek to the first entry in the column. + bool seek_to_first(); + + // Seek to the given ordinal entry in the column. + // Entry 0 is the first entry written to the column. + // If provided seek point is past the end of the file, + // then returns false. + bool seek_to_ordinal(rowid_t ord_idx) override; + + // Fetch the next vector of values from the page into 'dst'. + // The output vector must have space for up to n cells. + // + // return the size of entries. + // + // In the case that the values are themselves references + // to other memory (eg Slices), the referred-to memory is + // allocated in the dst column vector's arena. + virtual size_t next_vector(const size_t n, ColumnVector *dst) = 0; + + size_t get_current_oridinal(); + + // 每个批次读取的时候,会调用这个函数 + bool prepare_batch(size_t n); + + // 释放batch读取的相关资源 + bool finish_batch(); +}; + +} // namespace dfile + +} // namespace doris + +#endif // DORIS_BE_SRC_OLAP_ROWSET_DFILE_COLUMN_READER_H diff --git a/be/src/olap/rowset/dfile/column_writer.h b/be/src/olap/rowset/dfile/column_writer.h new file mode 100644 index 00000000000000..df855f0d2cf66f --- /dev/null +++ b/be/src/olap/rowset/dfile/column_writer.h @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_ROWSET_DFILE_COLUMN_WRITER_H +#define DORIS_BE_SRC_OLAP_ROWSET_DFILE_COLUMN_WRITER_H + +#include + +#include "gen_cpp/doris.pb.h" +#include "util/slice.h" + +namespace doris { + +namespace dfile { + +class ColumnWriter { +public: + explicit ColumnWriter(BuilderOptions builder_options, ColumnSchemaPB* column_schema) + : _builder_options(builder_options), + _column_schema(column_schema) { } + + bool init(); + + // close the writer + bool finish(); + + // 循环各个ColumnWriter, 通过以下几个接口,来构造对应的page pointer + // 之所以需要分这么多接口,是为了最终获取绝对位置,来构造page pointer + bool get_data_pages(std::vector* data_buffers); + + // Get the dictionary page for under dictionary encoding mode column. + virtual bool get_dictionary_page(Slice* dictionary_page); + + // Get the bloom filter page for under bloom filter indexed column. + virtual bool get_bloom_filter_pages(std::vector* bf_page); + + // Get the bitmap page for under bitmap indexed column. + virtual bool get_bitmap_page(Slice* bitmap_page); + + bool write_batch(RowBlock* block); + + size_t written_size() const; + + int written_value_count() const; + +private: + BuilderOptions _builder_options; + ColumnSchemaPB* _column_schema; +}; + +} // namespace dfile + +} // namespace doris + + +#endif // DORIS_BE_SRC_OLAP_ROWSET_DFILE_COLUMN_WRITER_H diff --git a/be/src/olap/rowset/dfile/common.h b/be/src/olap/rowset/dfile/common.h new file mode 100644 index 00000000000000..479b92b5f6d0ad --- /dev/null +++ b/be/src/olap/rowset/dfile/common.h @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_ROWSET_DFILE_COMMON_H +#define DORIS_BE_SRC_OLAP_ROWSET_DFILE_COMMON_H + +namespace doris { + +namespace dfile { + +typedef uint32_t rowid_t; + +} // namespace dfile + +} // namespace doris + +#endif // DORIS_BE_SRC_OLAP_ROWSET_DFILE_COMMON_H diff --git a/be/src/olap/rowset/dfile/options.h b/be/src/olap/rowset/dfile/options.h new file mode 100644 index 00000000000000..2e416518ea1411 --- /dev/null +++ b/be/src/olap/rowset/dfile/options.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_ROWSET_DFILE_OPTIONS_H +#define DORIS_BE_SRC_OLAP_ROWSET_DFILE_OPTIONS_H + +#include "gen_cpp/doris.pb.h" + +namespace doris { + +namespace dfile { + +struct BuilderOptions { + size_t data_page_size; + + size_t dict_page_size; + + bool write_posidx; + + EncodingTypePB encoding; + + CompressionTypePB compression_type; + + bool is_nullable; + + bool has_dictionary; +}; + +} // namespace dfile + +} // namespace doris + +#endif // DORIS_BE_SRC_OLAP_ROWSET_DFILE_OPTIONS_H diff --git a/be/src/olap/rowset/dfile/page_builder.h b/be/src/olap/rowset/dfile/page_builder.h new file mode 100644 index 00000000000000..6e619d42101925 --- /dev/null +++ b/be/src/olap/rowset/dfile/page_builder.h @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_ROWSET_DFILE_PAGE_BUILDER_H +#define DORIS_BE_SRC_OLAP_ROWSET_DFILE_PAGE_BUILDER_H + +#include +#include + +#include "util/slice.h" + +namespace doris { + +namespace dfile { + +class PageBuilder { +public: + virtual ~PageBuilder() { } + + // Used by column writer to determine whether the current page is full. + // Column writer depends on the result to decide whether to flush current page. + virtual bool is_page_full() = 0; + + // Get the dictionary page for under dictionary encoding mode column. + virtual bool get_dictionary_page(Slice* dictionary_page); + + // Get the bloom filter page for under bloom filter indexed column. + virtual bool get_bloom_filter_page(std::vector* bf_page); + + // Get the bitmap page for under bitmap indexed column. + virtual bool get_bitmap_page(Slice* bitmap_page); + + // Add a sequence of values to the page. + // Returns the number of values actually added, which may be less + // than requested if the page is full. + // + // vals size should be decided according to the page build type + virtual int add(const uint8_t* vals, size_t count) = 0; + + // Return a Slice which represents the encoded data of current page, + // And the page pointer to the page. The offset is relative to the current column. + // The offset of pointer should be revised in column writer. + // + // This Slice points to internal data of this builder. + virtual Slice finish(rowid_t first_page_rowid) = 0; + + // Reset the internal state of the page builder. + // + // Any data previously returned by finish may be invalidated by this call. + virtual void reset() = 0; + + // Return the number of entries that have been added to the page. + virtual size_t count() const = 0; + +private: + DISALLOW_COPY_AND_ASSIGN(PageBuilder); +}; + +} // namespace dfile + +} // namespace doris + +#endif // DORIS_BE_SRC_OLAP_ROWSET_DFILE_PAGE_BUILDER_H diff --git a/be/src/olap/rowset/dfile/page_decoder.h b/be/src/olap/rowset/dfile/page_decoder.h new file mode 100644 index 00000000000000..fd8763e4cbf226 --- /dev/null +++ b/be/src/olap/rowset/dfile/page_decoder.h @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_BE_SRC_OLAP_ROWSET_DFILE_PAGE_DECODER_H +#define DORIS_BE_SRC_OLAP_ROWSET_DFILE_PAGE_DECODER_H + +#include "runtime/vectorized_row_batch.h" + +namespace doris { + +namespace dfile { + +class PageDecoder { +public: + virtual ~PageDecoder() { } + + // Call this to do some preparation for decoder. + // eg: parse data block header + virtual bool init() = 0; + + // Seek the decoder to the given positional index of the page. + // For example, seek_to_position_in_block(0) seeks to the first + // stored entry. + // + // It is an error to call this with a value larger than Count(). + // Doing so has undefined results. + virtual void seek_to_position_in_block(size_t pos) = 0; + + // Seek the decoder forward by a given number of rows, or to the end + // of the page. This is primarily used to skip over data. + // + // Return the step skipped. + virtual size_t seek_forward(size_t n) { + size_t step = std::min(n, count() - current_index()); + DCHECK_GE(step, 0); + seek_to_position_in_block(current_index() + step); + return step; + } + + // Fetch the next vector of values from the page into 'dst'. + // The output vector must have space for up to n cells. + // + // return the size of entries. + // + // In the case that the values are themselves references + // to other memory (eg Slices), the referred-to memory is + // allocated in the dst column vector's arena. + virtual size_t next_vector(const size_t n, ColumnVector *dst) = 0; + + // Return the number of elements in this page. + virtual size_t count() const = 0; + + // Return the position within the page of the currently seeked + // entry (ie the entry that will next be returned by next_vector()) + virtual size_t current_index() const = 0; + + // Return the first rowid stored in this page. + virtual rowid_t get_first_rowid() const = 0; + +private: + DISALLOW_COPY_AND_ASSIGN(PageDecoder); +}; + +} // namespace dfile + +} // namespace doris + +#endif // DORIS_BE_SRC_OLAP_ROWSET_DFILE_PAGE_DECODER_H diff --git a/gensrc/proto/doris.proto b/gensrc/proto/doris.proto new file mode 100644 index 00000000000000..3edebfce8c1430 --- /dev/null +++ b/gensrc/proto/doris.proto @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// Define file format struct, like data header, index header. + +package doris; + +message ColumnSchemaPB { + optional uint32 column_id = 1; + optional string type = 2; + optional string aggregation = 3; + optional uint32 length = 4; + optional bool is_key = 5; + optional string default_value = 6; + optional uint32 precision = 9 [default = 27]; + optional uint32 frac = 10 [default = 9]; + optional bool is_nullable = 11 [default=false]; + optional bool is_bf_column = 15 [default=false]; // is bloom filter indexed column + optional bool is_bitmap_column = 16 [default=false]; +} + +// page position info +message PagePointerPB { + required uint64 offset = 1; // offset in segment file + required uint32 length = 2; // size of page in byte +} + +message MetadataPairPB { + optional string key = 1; + optional bytes value = 2; +} + +enum EncodingTypePB { + PLAIN_ENCODING = 1; + PREFIX_ENCODING = 2; + RLE = 4; + DICT_ENCODING = 5; + BIT_SHUFFLE = 6; + UNKNOWN_ENCODING = 1000; +} + +enum CompressionTypePB { + DEFAULT_COMPRESSION = 0; + NO_COMPRESSION = 1; + SNAPPY = 2; + LZ4 = 3; + ZLIB = 4; + ZSTB = 5; + LZO = 6; + UNKNOWN_COMPRESSION = 1000; +} + +message ZoneMapPB { + optional bytes min = 1; + optional bytes max = 2; + optional bool null_flag = 3; +} + +message ColumnMetaPB { + optional EncodingTypePB encoding = 1; + + optional PagePointerPB dict_page = 2;// dictionary page for DICT_ENCODING + repeated PagePointerPB bloom_filter_pages = 3; // bloom filter pages for bloom filter column + optional PagePointerPB ordinal_index_page = 4; // ordinal index page + optional PagePointerPB page_zonemap_page = 5; // page zonemap info of column + + optional PagePointerPB bitmap_index_page = 6; // bitmap index page + + optional uint64 data_footprint = 7; // data footprint of column after encoding and compress + optional uint64 index_footprint = 8; // index footprint of column after encoding and compress + optional uint64 raw_data_footprint = 9; // raw column data footprint + + optional CompressionTypePB compress_type = 10; // compress type for column + + optional ZoneMapPB column_zonemap = 11; // column zonemap info + repeated MetadataPairPB column_meta_datas = 12; +} + +message FileFooterPB { + optional uint32 version = 1 [default = 1]; // file version + repeated ColumnSchemaPB schema = 2; // tablet schema + optional uint64 num_values = 3; // number of values + optional uint64 index_footprint = 4; // total idnex footprint of all columns + optional uint64 data_footprint = 5; // total data footprint of all columns + optional uint64 raw_data_footprint = 6; // raw data footprint + + optional CompressionTypePB compress_type = 7 [default = LZO]; // default compression type for file columns + repeated MetadataPairPB file_meta_datas = 8; // meta data of file + optional PagePointerPB key_index_page = 9; // short key index page +}