From 000d19f8328788b6ab49a3924dbdb991b27fa18e Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 4 Jun 2020 16:53:20 -0500 Subject: [PATCH 1/7] Initial splitting up Some cleanup clang-format Fixes, move GenerateBits/Unrolled to separate header Fix R compilation Fix rebase conflict --- cpp/build-support/iwyu/iwyu.sh | 2 +- cpp/src/arrow/CMakeLists.txt | 3 + cpp/src/arrow/array/array_dict.cc | 1 + cpp/src/arrow/array/array_nested.cc | 1 + cpp/src/arrow/array/builder_union.h | 2 +- cpp/src/arrow/array/concatenate.cc | 1 + cpp/src/arrow/array/data.cc | 2 +- cpp/src/arrow/array/data.h | 1 - cpp/src/arrow/array/diff.h | 1 - cpp/src/arrow/buffer_builder.h | 1 + cpp/src/arrow/compare.cc | 1 + cpp/src/arrow/compute/api_scalar.h | 1 - cpp/src/arrow/compute/exec.cc | 1 + cpp/src/arrow/compute/exec_test.cc | 1 + .../arrow/compute/kernels/codegen_internal.h | 3 + cpp/src/arrow/compute/kernels/common.h | 5 - .../arrow/compute/kernels/scalar_boolean.cc | 6 + .../compute/kernels/scalar_cast_string.cc | 4 + .../compute/kernels/scalar_cast_temporal.cc | 3 +- .../compute/kernels/scalar_compare_test.cc | 8 +- .../compute/kernels/scalar_set_lookup.cc | 3 + .../arrow/compute/kernels/vector_filter.cc | 1 + cpp/src/arrow/ipc/writer.cc | 1 + cpp/src/arrow/json/parser.cc | 1 + cpp/src/arrow/python/numpy_to_arrow.cc | 2 + cpp/src/arrow/util/bit_block_counter.cc | 75 ++ cpp/src/arrow/util/bit_block_counter.h | 55 ++ cpp/src/arrow/util/bit_util.cc | 534 ------------ cpp/src/arrow/util/bit_util.h | 797 +----------------- cpp/src/arrow/util/bit_util_benchmark.cc | 6 + cpp/src/arrow/util/bit_util_test.cc | 6 + cpp/src/arrow/util/bitmap.cc | 65 ++ cpp/src/arrow/util/bitmap.h | 294 +++++++ cpp/src/arrow/util/bitmap_generate.h | 112 +++ cpp/src/arrow/util/bitmap_ops.cc | 495 +++++++++++ cpp/src/arrow/util/bitmap_ops.h | 223 +++++ cpp/src/arrow/util/bitmap_reader.h | 89 ++ cpp/src/arrow/util/bitmap_writer.h | 202 +++++ cpp/src/arrow/util/bitset.h | 89 ++ cpp/src/arrow/util/hashing.h | 1 + cpp/src/arrow/util/rle_encoding.h | 1 + cpp/src/arrow/visitor_inline.h | 1 + cpp/src/gandiva/bitmap_accumulator.cc | 2 + cpp/src/gandiva/bitmap_accumulator_test.cc | 1 + cpp/src/parquet/arrow/path_internal.cc | 1 + cpp/src/parquet/level_conversion_test.cc | 2 + cpp/src/parquet/platform.h | 19 +- cpp/src/parquet/statistics.cc | 1 + cpp/src/parquet/statistics_test.cc | 1 + r/src/array.cpp | 1 + r/src/array_from_vector.cpp | 1 + r/src/array_to_vector.cpp | 2 + 52 files changed, 1777 insertions(+), 1355 deletions(-) create mode 100644 cpp/src/arrow/util/bit_block_counter.cc create mode 100644 cpp/src/arrow/util/bit_block_counter.h create mode 100644 cpp/src/arrow/util/bitmap.cc create mode 100644 cpp/src/arrow/util/bitmap.h create mode 100644 cpp/src/arrow/util/bitmap_generate.h create mode 100644 cpp/src/arrow/util/bitmap_ops.cc create mode 100644 cpp/src/arrow/util/bitmap_ops.h create mode 100644 cpp/src/arrow/util/bitmap_reader.h create mode 100644 cpp/src/arrow/util/bitmap_writer.h create mode 100644 cpp/src/arrow/util/bitset.h diff --git a/cpp/build-support/iwyu/iwyu.sh b/cpp/build-support/iwyu/iwyu.sh index 27c62dcd9b1..813f64d6bc4 100755 --- a/cpp/build-support/iwyu/iwyu.sh +++ b/cpp/build-support/iwyu/iwyu.sh @@ -25,7 +25,7 @@ IWYU_LOG=$(mktemp -t arrow-cpp-iwyu.XXXXXX) trap "rm -f $IWYU_LOG" EXIT IWYU_MAPPINGS_PATH="$ROOT/cpp/build-support/iwyu/mappings" -IWYU_ARGS="--no_fwd_decls \ +IWYU_ARGS="\ --mapping_file=$IWYU_MAPPINGS_PATH/boost-all.imp \ --mapping_file=$IWYU_MAPPINGS_PATH/boost-all-private.imp \ --mapping_file=$IWYU_MAPPINGS_PATH/boost-extra.imp \ diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index f622cd86a33..cac6b820781 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -173,7 +173,10 @@ set(ARROW_SRCS io/slow.cc testing/util.cc util/basic_decimal.cc + util/bit_block_counter.cc util/bit_util.cc + util/bitmap.cc + util/bitmap_ops.cc util/compression.cc util/cpu_info.cc util/decimal.cc diff --git a/cpp/src/arrow/array/array_dict.cc b/cpp/src/arrow/array/array_dict.cc index 1c5abf6e0d8..48e98bd11f2 100644 --- a/cpp/src/arrow/array/array_dict.cc +++ b/cpp/src/arrow/array/array_dict.cc @@ -33,6 +33,7 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" #include "arrow/util/int_util.h" #include "arrow/util/logging.h" diff --git a/cpp/src/arrow/array/array_nested.cc b/cpp/src/arrow/array/array_nested.cc index fa07694420a..7e4a71ab344 100644 --- a/cpp/src/arrow/array/array_nested.cc +++ b/cpp/src/arrow/array/array_nested.cc @@ -34,6 +34,7 @@ #include "arrow/type_traits.h" #include "arrow/util/atomic_shared_ptr.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" #include "arrow/util/logging.h" diff --git a/cpp/src/arrow/array/builder_union.h b/cpp/src/arrow/array/builder_union.h index 0f8f258247c..aba707e8383 100644 --- a/cpp/src/arrow/array/builder_union.h +++ b/cpp/src/arrow/array/builder_union.h @@ -20,13 +20,13 @@ #include #include #include -#include #include #include "arrow/array/array_nested.h" #include "arrow/array/builder_base.h" #include "arrow/array/data.h" #include "arrow/buffer_builder.h" +#include "arrow/memory_pool.h" #include "arrow/status.h" #include "arrow/type.h" #include "arrow/util/visibility.h" diff --git a/cpp/src/arrow/array/concatenate.cc b/cpp/src/arrow/array/concatenate.cc index e83bc47a889..e1f6d71dc55 100644 --- a/cpp/src/arrow/array/concatenate.cc +++ b/cpp/src/arrow/array/concatenate.cc @@ -34,6 +34,7 @@ #include "arrow/status.h" #include "arrow/type.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" #include "arrow/util/logging.h" #include "arrow/visitor_inline.h" diff --git a/cpp/src/arrow/array/data.cc b/cpp/src/arrow/array/data.cc index 3aa6db62812..6bd7f1eb450 100644 --- a/cpp/src/arrow/array/data.cc +++ b/cpp/src/arrow/array/data.cc @@ -28,7 +28,7 @@ #include "arrow/buffer.h" #include "arrow/status.h" #include "arrow/type.h" -#include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/logging.h" #include "arrow/util/macros.h" diff --git a/cpp/src/arrow/array/data.h b/cpp/src/arrow/array/data.h index afe6694b846..23f46dfc761 100644 --- a/cpp/src/arrow/array/data.h +++ b/cpp/src/arrow/array/data.h @@ -25,7 +25,6 @@ #include "arrow/buffer.h" #include "arrow/result.h" -#include "arrow/type_fwd.h" #include "arrow/util/macros.h" #include "arrow/util/visibility.h" diff --git a/cpp/src/arrow/array/diff.h b/cpp/src/arrow/array/diff.h index 00565a1f577..a405164b333 100644 --- a/cpp/src/arrow/array/diff.h +++ b/cpp/src/arrow/array/diff.h @@ -27,7 +27,6 @@ #include "arrow/result.h" #include "arrow/status.h" #include "arrow/type.h" -#include "arrow/util/bit_util.h" #include "arrow/util/visibility.h" namespace arrow { diff --git a/cpp/src/arrow/buffer_builder.h b/cpp/src/arrow/buffer_builder.h index 1870ee6e996..41a47c91729 100644 --- a/cpp/src/arrow/buffer_builder.h +++ b/cpp/src/arrow/buffer_builder.h @@ -27,6 +27,7 @@ #include "arrow/buffer.h" #include "arrow/status.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_generate.h" #include "arrow/util/macros.h" #include "arrow/util/ubsan.h" #include "arrow/util/visibility.h" diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc index f1af4184fd7..db4033863fc 100644 --- a/cpp/src/arrow/compare.cc +++ b/cpp/src/arrow/compare.cc @@ -39,6 +39,7 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" #include "arrow/util/logging.h" #include "arrow/util/macros.h" diff --git a/cpp/src/arrow/compute/api_scalar.h b/cpp/src/arrow/compute/api_scalar.h index 66c29f495a0..e3210d3ac91 100644 --- a/cpp/src/arrow/compute/api_scalar.h +++ b/cpp/src/arrow/compute/api_scalar.h @@ -20,7 +20,6 @@ #pragma once -#include #include #include diff --git a/cpp/src/arrow/compute/exec.cc b/cpp/src/arrow/compute/exec.cc index b1828572308..c5581bfacf2 100644 --- a/cpp/src/arrow/compute/exec.cc +++ b/cpp/src/arrow/compute/exec.cc @@ -40,6 +40,7 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" #include "arrow/util/cpu_info.h" #include "arrow/util/logging.h" diff --git a/cpp/src/arrow/compute/exec_test.cc b/cpp/src/arrow/compute/exec_test.cc index f938a45cf01..d8615075054 100644 --- a/cpp/src/arrow/compute/exec_test.cc +++ b/cpp/src/arrow/compute/exec_test.cc @@ -38,6 +38,7 @@ #include "arrow/table.h" #include "arrow/type.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" #include "arrow/util/cpu_info.h" #include "arrow/util/logging.h" diff --git a/cpp/src/arrow/compute/kernels/codegen_internal.h b/cpp/src/arrow/compute/kernels/codegen_internal.h index b14381b9483..de420017ec5 100644 --- a/cpp/src/arrow/compute/kernels/codegen_internal.h +++ b/cpp/src/arrow/compute/kernels/codegen_internal.h @@ -34,6 +34,9 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_generate.h" +#include "arrow/util/bitmap_reader.h" +#include "arrow/util/bitmap_writer.h" #include "arrow/util/decimal.h" #include "arrow/util/logging.h" #include "arrow/util/optional.h" diff --git a/cpp/src/arrow/compute/kernels/common.h b/cpp/src/arrow/compute/kernels/common.h index 23557876cf2..b2e02cbec4e 100644 --- a/cpp/src/arrow/compute/kernels/common.h +++ b/cpp/src/arrow/compute/kernels/common.h @@ -39,21 +39,16 @@ #include "arrow/table.h" #include "arrow/type.h" #include "arrow/type_traits.h" -#include "arrow/util/bit_util.h" #include "arrow/util/checked_cast.h" #include "arrow/util/logging.h" #include "arrow/util/macros.h" #include "arrow/util/string_view.h" -#include "arrow/visitor_inline.h" // IWYU pragma: end_exports namespace arrow { -using internal::Bitmap; -using internal::BitmapReader; using internal::checked_cast; using internal::checked_pointer_cast; -using internal::FirstTimeBitmapWriter; } // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/scalar_boolean.cc b/cpp/src/arrow/compute/kernels/scalar_boolean.cc index 15ff8cbe309..89f4de08052 100644 --- a/cpp/src/arrow/compute/kernels/scalar_boolean.cc +++ b/cpp/src/arrow/compute/kernels/scalar_boolean.cc @@ -18,8 +18,14 @@ #include #include "arrow/compute/kernels/common.h" +#include "arrow/util/bit_util.h" +#include "arrow/util/bitmap.h" +#include "arrow/util/bitmap_ops.h" namespace arrow { + +using internal::Bitmap; + namespace compute { namespace { diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_string.cc b/cpp/src/arrow/compute/kernels/scalar_cast_string.cc index 66182604183..9830102b6aa 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_string.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_string.cc @@ -17,10 +17,14 @@ // Implementation of casting to integer or floating point types +#include "arrow/array/array_base.h" #include "arrow/compute/kernels/common.h" #include "arrow/compute/kernels/scalar_cast_internal.h" +#include "arrow/result.h" #include "arrow/util/formatting.h" +#include "arrow/util/optional.h" #include "arrow/util/utf8.h" +#include "arrow/visitor_inline.h" namespace arrow { diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc b/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc index a793504b2dc..b2addd04e1f 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc @@ -18,11 +18,10 @@ // Implementation of casting to (or between) temporal types #include -#include -#include #include "arrow/compute/kernels/common.h" #include "arrow/compute/kernels/scalar_cast_internal.h" +#include "arrow/util/bitmap_reader.h" #include "arrow/util/time.h" #include "arrow/util/value_parsing.h" diff --git a/cpp/src/arrow/compute/kernels/scalar_compare_test.cc b/cpp/src/arrow/compute/kernels/scalar_compare_test.cc index df4306d94b0..758e10b60d8 100644 --- a/cpp/src/arrow/compute/kernels/scalar_compare_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_compare_test.cc @@ -27,13 +27,13 @@ #include "arrow/array.h" #include "arrow/compute/api.h" #include "arrow/compute/kernels/test_util.h" -#include "arrow/type.h" -#include "arrow/type_traits.h" -#include "arrow/util/checked_cast.h" - #include "arrow/testing/gtest_common.h" #include "arrow/testing/gtest_util.h" #include "arrow/testing/random.h" +#include "arrow/type.h" +#include "arrow/type_traits.h" +#include "arrow/util/bitmap_reader.h" +#include "arrow/util/checked_cast.h" namespace arrow { namespace compute { diff --git a/cpp/src/arrow/compute/kernels/scalar_set_lookup.cc b/cpp/src/arrow/compute/kernels/scalar_set_lookup.cc index bdc815ab8a7..fc14bf5e1d4 100644 --- a/cpp/src/arrow/compute/kernels/scalar_set_lookup.cc +++ b/cpp/src/arrow/compute/kernels/scalar_set_lookup.cc @@ -19,8 +19,11 @@ #include "arrow/array/builder_primitive.h" #include "arrow/compute/api_scalar.h" #include "arrow/compute/kernels/common.h" +#include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_writer.h" #include "arrow/util/hashing.h" #include "arrow/util/optional.h" +#include "arrow/visitor_inline.h" namespace arrow { diff --git a/cpp/src/arrow/compute/kernels/vector_filter.cc b/cpp/src/arrow/compute/kernels/vector_filter.cc index 03a725d2bbf..db21d402e35 100644 --- a/cpp/src/arrow/compute/kernels/vector_filter.cc +++ b/cpp/src/arrow/compute/kernels/vector_filter.cc @@ -22,6 +22,7 @@ #include "arrow/compute/kernels/vector_selection_internal.h" #include "arrow/record_batch.h" #include "arrow/result.h" +#include "arrow/visitor_inline.h" namespace arrow { namespace compute { diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc index c8bb9a336b4..ae0af10791e 100644 --- a/cpp/src/arrow/ipc/writer.cc +++ b/cpp/src/arrow/ipc/writer.cc @@ -45,6 +45,7 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" #include "arrow/util/compression.h" #include "arrow/util/key_value_metadata.h" diff --git a/cpp/src/arrow/json/parser.cc b/cpp/src/arrow/json/parser.cc index 334af097699..bca6a35b667 100644 --- a/cpp/src/arrow/json/parser.cc +++ b/cpp/src/arrow/json/parser.cc @@ -33,6 +33,7 @@ #include "arrow/builder.h" #include "arrow/memory_pool.h" #include "arrow/type.h" +#include "arrow/util/bitset.h" #include "arrow/util/logging.h" #include "arrow/util/make_unique.h" #include "arrow/util/string_view.h" diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc index 329fb310910..76a0ac8eff9 100644 --- a/cpp/src/arrow/python/numpy_to_arrow.cc +++ b/cpp/src/arrow/python/numpy_to_arrow.cc @@ -36,6 +36,8 @@ #include "arrow/type_fwd.h" #include "arrow/type_traits.h" #include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_generate.h" +#include "arrow/util/bitmap_ops.h" #include "arrow/util/checked_cast.h" #include "arrow/util/logging.h" #include "arrow/util/macros.h" diff --git a/cpp/src/arrow/util/bit_block_counter.cc b/cpp/src/arrow/util/bit_block_counter.cc new file mode 100644 index 00000000000..a596c07ef85 --- /dev/null +++ b/cpp/src/arrow/util/bit_block_counter.cc @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/util/bit_block_counter.h" + +#include +#include + +#include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_ops.h" +#include "arrow/util/ubsan.h" + +namespace arrow { +namespace internal { + +BitBlockCounter::Block BitBlockCounter::NextBlock() { + auto load_word = [](const uint8_t* bytes) -> uint64_t { + return BitUtil::ToLittleEndian(util::SafeLoadAs(bytes)); + }; + auto shift_word = [](uint64_t current, uint64_t next, int64_t shift) -> uint64_t { + return (current >> shift) | (next << (64 - shift)); + }; + + // When the offset is > 0, we need there to be a word beyond the last aligned + // word in the bitmap for the bit shifting logic. + const int64_t bits_required_to_scan_words = offset_ == 0 ? 256 : 256 + (64 - offset_); + if (bits_remaining_ < bits_required_to_scan_words) { + // End of the bitmap, leave it to the caller to decide how to best check + // these bits, no need to do redundant computation here. + const int16_t run_length = static_cast(bits_remaining_); + bits_remaining_ -= run_length; + return {run_length, static_cast(CountSetBits(bitmap_, offset_, run_length))}; + } + + int64_t total_popcount = 0; + if (offset_ == 0) { + total_popcount += BitUtil::PopCount(load_word(bitmap_)); + total_popcount += BitUtil::PopCount(load_word(bitmap_ + 8)); + total_popcount += BitUtil::PopCount(load_word(bitmap_ + 16)); + total_popcount += BitUtil::PopCount(load_word(bitmap_ + 24)); + } else { + auto current = load_word(bitmap_); + auto next = load_word(bitmap_ + 8); + total_popcount += BitUtil::PopCount(shift_word(current, next, offset_)); + current = next; + next = load_word(bitmap_ + 16); + total_popcount += BitUtil::PopCount(shift_word(current, next, offset_)); + current = next; + next = load_word(bitmap_ + 24); + total_popcount += BitUtil::PopCount(shift_word(current, next, offset_)); + current = next; + next = load_word(bitmap_ + 32); + total_popcount += BitUtil::PopCount(shift_word(current, next, offset_)); + } + bitmap_ += BitUtil::BytesForBits(kTargetBlockLength); + bits_remaining_ -= 256; + return {256, static_cast(total_popcount)}; +} + +} // namespace internal +} // namespace arrow diff --git a/cpp/src/arrow/util/bit_block_counter.h b/cpp/src/arrow/util/bit_block_counter.h new file mode 100644 index 00000000000..92e2b9d3aa4 --- /dev/null +++ b/cpp/src/arrow/util/bit_block_counter.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/util/visibility.h" + +namespace arrow { +namespace internal { + +/// \brief A class that scans through a true/false bitmap to yield blocks of up +/// to 256 bits at a time along with their popcount. This is used to accelerate +/// processing of mostly-not-null array data. +class ARROW_EXPORT BitBlockCounter { + public: + struct Block { + int16_t length; + int16_t popcount; + }; + + static constexpr int16_t kTargetBlockLength = 256; + + BitBlockCounter(const uint8_t* bitmap, int64_t start_offset, int64_t length) + : bitmap_(bitmap + start_offset / 8), + bits_remaining_(length), + offset_(start_offset % 8) {} + + /// \brief Return the next run of available bits, up to 256. The returned + /// pair contains the size of run and the number of true values + Block NextBlock(); + + private: + const uint8_t* bitmap_; + int64_t bits_remaining_; + int64_t offset_; +}; + +} // namespace internal +} // namespace arrow diff --git a/cpp/src/arrow/util/bit_util.cc b/cpp/src/arrow/util/bit_util.cc index 8b63da6b2df..ace3c65efc8 100644 --- a/cpp/src/arrow/util/bit_util.cc +++ b/cpp/src/arrow/util/bit_util.cc @@ -101,538 +101,4 @@ Result> BytesToBits(const std::vector& bytes, } } // namespace BitUtil - -namespace internal { - -int64_t CountSetBits(const uint8_t* data, int64_t bit_offset, int64_t length) { - constexpr int64_t pop_len = sizeof(uint64_t) * 8; - DCHECK_GE(bit_offset, 0); - int64_t count = 0; - - const auto p = BitmapWordAlign(data, bit_offset, length); - for (int64_t i = bit_offset; i < bit_offset + p.leading_bits; ++i) { - if (BitUtil::GetBit(data, i)) { - ++count; - } - } - - if (p.aligned_words > 0) { - // popcount as much as possible with the widest possible count - const uint64_t* u64_data = reinterpret_cast(p.aligned_start); - DCHECK_EQ(reinterpret_cast(u64_data) & 7, 0); - const uint64_t* end = u64_data + p.aligned_words; - - for (auto iter = u64_data; iter < end; ++iter) { - count += BitUtil::PopCount(*iter); - } - } - - // Account for left over bits (in theory we could fall back to smaller - // versions of popcount but the code complexity is likely not worth it) - for (int64_t i = p.trailing_bit_offset; i < bit_offset + length; ++i) { - if (BitUtil::GetBit(data, i)) { - ++count; - } - } - - return count; -} - -template -void TransferBitmap(const uint8_t* data, int64_t offset, int64_t length, - int64_t dest_offset, uint8_t* dest) { - int64_t byte_offset = offset / 8; - int64_t bit_offset = offset % 8; - int64_t dest_byte_offset = dest_offset / 8; - int64_t dest_bit_offset = dest_offset % 8; - int64_t num_bytes = BitUtil::BytesForBits(length); - // Shift dest by its byte offset - dest += dest_byte_offset; - - if (bit_offset || dest_bit_offset) { - data += byte_offset; - - const int64_t n_words = length / 64; - if (n_words > 1) { - auto load_word = [](const uint8_t* bytes) -> uint64_t { - return BitUtil::ToLittleEndian(util::SafeLoadAs(bytes)); - }; - auto shift_word = [](uint64_t current, uint64_t next, int64_t shift) -> uint64_t { - if (shift == 0) return current; - return (current >> shift) | (next << (64 - shift)); - }; - auto write_word = [](uint8_t* bytes, uint64_t word) { - util::SafeStore(bytes, BitUtil::FromLittleEndian(word)); - }; - - const uint64_t dest_mask = (1U << dest_bit_offset) - 1; - auto data_current = load_word(data); - auto dest_current = load_word(dest); - - for (int64_t i = 0; i < n_words - 1; ++i) { - data += 8; - const auto data_next = load_word(data); - auto word = shift_word(data_current, data_next, bit_offset); - data_current = data_next; - if (invert_bits) { - word = ~word; - } - - if (dest_bit_offset) { - word = (word << dest_bit_offset) | (word >> (64 - dest_bit_offset)); - auto dest_next = load_word(dest + 8); - dest_current = (dest_current & dest_mask) | (word & ~dest_mask); - dest_next = (dest_next & ~dest_mask) | (word & dest_mask); - write_word(dest, dest_current); - write_word(dest + 8, dest_next); - dest_current = dest_next; - } else { - write_word(dest, word); - } - dest += 8; - } - - length -= (n_words - 1) * 64; - } - - internal::BitmapReader valid_reader(data, bit_offset, length); - internal::BitmapWriter valid_writer(dest, dest_bit_offset, length); - - for (int64_t i = 0; i < length; i++) { - if (invert_bits ^ valid_reader.IsSet()) { - valid_writer.Set(); - } else { - valid_writer.Clear(); - } - valid_reader.Next(); - valid_writer.Next(); - } - valid_writer.Finish(); - } else { - // Take care of the trailing bits in the last byte - int64_t trailing_bits = num_bytes * 8 - length; - uint8_t trail = 0; - if (trailing_bits && restore_trailing_bits) { - trail = dest[num_bytes - 1]; - } - - if (invert_bits) { - for (int64_t i = 0; i < num_bytes; i++) { - dest[i] = static_cast(~(data[byte_offset + i])); - } - } else { - std::memcpy(dest, data + byte_offset, static_cast(num_bytes)); - } - - if (restore_trailing_bits) { - for (int i = 0; i < trailing_bits; i++) { - if (BitUtil::GetBit(&trail, i + 8 - trailing_bits)) { - BitUtil::SetBit(dest, length + i); - } else { - BitUtil::ClearBit(dest, length + i); - } - } - } - } -} - -template -Result> TransferBitmap(MemoryPool* pool, const uint8_t* data, - int64_t offset, int64_t length) { - ARROW_ASSIGN_OR_RAISE(auto buffer, AllocateEmptyBitmap(length, pool)); - uint8_t* dest = buffer->mutable_data(); - - TransferBitmap(data, offset, length, 0, dest); - - // As we have freshly allocated this bitmap, we should take care of zeroing the - // remaining bits. - int64_t num_bytes = BitUtil::BytesForBits(length); - int64_t bits_to_zero = num_bytes * 8 - length; - for (int64_t i = length; i < length + bits_to_zero; ++i) { - // Both branches may copy extra bits - unsetting to match specification. - BitUtil::ClearBit(dest, i); - } - return buffer; -} - -void CopyBitmap(const uint8_t* data, int64_t offset, int64_t length, uint8_t* dest, - int64_t dest_offset, bool restore_trailing_bits) { - if (restore_trailing_bits) { - TransferBitmap(data, offset, length, dest_offset, dest); - } else { - TransferBitmap(data, offset, length, dest_offset, dest); - } -} - -void InvertBitmap(const uint8_t* data, int64_t offset, int64_t length, uint8_t* dest, - int64_t dest_offset) { - TransferBitmap(data, offset, length, dest_offset, dest); -} - -Result> CopyBitmap(MemoryPool* pool, const uint8_t* data, - int64_t offset, int64_t length) { - return TransferBitmap(pool, data, offset, length); -} - -Result> InvertBitmap(MemoryPool* pool, const uint8_t* data, - int64_t offset, int64_t length, - std::shared_ptr* out) { - return TransferBitmap(pool, data, offset, length); -} - -bool BitmapEquals(const uint8_t* left, int64_t left_offset, const uint8_t* right, - int64_t right_offset, int64_t bit_length) { - if (left_offset % 8 == 0 && right_offset % 8 == 0) { - // byte aligned, can use memcmp - bool bytes_equal = std::memcmp(left + left_offset / 8, right + right_offset / 8, - bit_length / 8) == 0; - if (!bytes_equal) { - return false; - } - for (int64_t i = (bit_length / 8) * 8; i < bit_length; ++i) { - if (BitUtil::GetBit(left, left_offset + i) != - BitUtil::GetBit(right, right_offset + i)) { - return false; - } - } - return true; - } - - // Unaligned slow case - left += left_offset / 8; - right += right_offset / 8; - left_offset %= 8; - right_offset %= 8; - - // process in 64 bits, may touch two adjacent words in one iteration - const int64_t n_words = bit_length / 64; - if (n_words > 1) { - auto load_word = [](const uint8_t* bytes) -> uint64_t { - return BitUtil::ToLittleEndian(util::SafeLoadAs(bytes)); - }; - auto shift_word = [](uint64_t current, uint64_t next, int64_t shift) -> uint64_t { - if (shift == 0) return current; - return (current >> shift) | (next << (64 - shift)); - }; - - auto left_current = load_word(left); - auto right_current = load_word(right); - - for (int64_t i = 0; i < n_words - 1; ++i) { - left += 8; - auto left_next = load_word(left); - auto left_word = shift_word(left_current, left_next, left_offset); - left_current = left_next; - - right += 8; - auto right_next = load_word(right); - auto right_word = shift_word(right_current, right_next, right_offset); - right_current = right_next; - - if (left_word != right_word) { - return false; - } - } - - bit_length -= (n_words - 1) * 64; - } - - // process in bit - for (int64_t i = 0; i < bit_length; ++i) { - if (BitUtil::GetBit(left, left_offset + i) != - BitUtil::GetBit(right, right_offset + i)) { - return false; - } - } - return true; -} - -namespace { - -template