From e317fb4cbe45d2dcf4d669e0d2786253521801fe Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 5 Jun 2020 12:16:52 -0500 Subject: [PATCH 1/4] Expand / improve Take and Filter benchmarks for enhanced baseline --- cpp/src/arrow/compute/benchmark_util.h | 7 +- cpp/src/arrow/compute/kernels/CMakeLists.txt | 3 +- .../kernels/vector_filter_benchmark.cc | 107 ------- .../kernels/vector_selection_benchmark.cc | 270 ++++++++++++++++++ .../compute/kernels/vector_take_benchmark.cc | 146 ---------- 5 files changed, 275 insertions(+), 258 deletions(-) delete mode 100644 cpp/src/arrow/compute/kernels/vector_filter_benchmark.cc create mode 100644 cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc delete mode 100644 cpp/src/arrow/compute/kernels/vector_take_benchmark.cc diff --git a/cpp/src/arrow/compute/benchmark_util.h b/cpp/src/arrow/compute/benchmark_util.h index dbc206c5f24..d72484580df 100644 --- a/cpp/src/arrow/compute/benchmark_util.h +++ b/cpp/src/arrow/compute/benchmark_util.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include "arrow/testing/gtest_util.h" @@ -56,7 +57,7 @@ void BenchmarkSetArgsWithSizes(benchmark::internal::Benchmark* bench, bench->Unit(benchmark::kMicrosecond); for (auto size : sizes) - for (auto nulls : std::vector({0, 1, 10, 50})) + for (auto nulls : std::vector({10000, 1000, 100, 50, 10, 1})) bench->Args({static_cast(size), nulls}); } @@ -80,12 +81,12 @@ struct RegressionArgs { explicit RegressionArgs(benchmark::State& state) : size(state.range(0)), - null_proportion(static_cast(state.range(1)) / 100.0), + null_proportion(1. / static_cast(state.range(1))), state_(state) {} ~RegressionArgs() { state_.counters["size"] = static_cast(size); - state_.counters["null_percent"] = static_cast(state_.range(1)); + state_.counters["null_percent"] = null_proportion * 100; state_.SetBytesProcessed(state_.iterations() * size); } diff --git a/cpp/src/arrow/compute/kernels/CMakeLists.txt b/cpp/src/arrow/compute/kernels/CMakeLists.txt index 9eb23716a06..bf4ecff9355 100644 --- a/cpp/src/arrow/compute/kernels/CMakeLists.txt +++ b/cpp/src/arrow/compute/kernels/CMakeLists.txt @@ -44,8 +44,7 @@ add_arrow_compute_test(vector_test add_arrow_benchmark(vector_hash_benchmark PREFIX "arrow-compute") add_arrow_benchmark(vector_sort_benchmark PREFIX "arrow-compute") add_arrow_benchmark(vector_partition_benchmark PREFIX "arrow-compute") -add_arrow_benchmark(vector_filter_benchmark PREFIX "arrow-compute") -add_arrow_benchmark(vector_take_benchmark PREFIX "arrow-compute") +add_arrow_benchmark(vector_selection_benchmark PREFIX "arrow-compute") # ---------------------------------------------------------------------- # Aggregate kernels diff --git a/cpp/src/arrow/compute/kernels/vector_filter_benchmark.cc b/cpp/src/arrow/compute/kernels/vector_filter_benchmark.cc deleted file mode 100644 index 85bb58d6b96..00000000000 --- a/cpp/src/arrow/compute/kernels/vector_filter_benchmark.cc +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "benchmark/benchmark.h" - -#include "arrow/compute/api_vector.h" -#include "arrow/compute/benchmark_util.h" -#include "arrow/compute/kernels/test_util.h" -#include "arrow/testing/gtest_util.h" -#include "arrow/testing/random.h" - -namespace arrow { -namespace compute { - -constexpr auto kSeed = 0x0ff1ce; - -static void FilterInt64(benchmark::State& state) { - RegressionArgs args(state); - - const int64_t array_size = args.size / sizeof(int64_t); - auto rand = random::RandomArrayGenerator(kSeed); - auto array = std::static_pointer_cast>( - rand.Int64(array_size, -100, 100, args.null_proportion)); - auto filter = std::static_pointer_cast( - rand.Boolean(array_size, 0.75, args.null_proportion)); - - for (auto _ : state) { - ABORT_NOT_OK(Filter(array, filter).status()); - } -} - -static void FilterFixedSizeList1Int64(benchmark::State& state) { - RegressionArgs args(state); - - const int64_t array_size = args.size / sizeof(int64_t); - auto rand = random::RandomArrayGenerator(kSeed); - auto int_array = std::static_pointer_cast>( - rand.Int64(array_size, -100, 100, args.null_proportion)); - auto array = std::make_shared( - fixed_size_list(int64(), 1), array_size, int_array, int_array->null_bitmap(), - int_array->null_count()); - auto filter = std::static_pointer_cast( - rand.Boolean(array_size, 0.75, args.null_proportion)); - - for (auto _ : state) { - ABORT_NOT_OK(Filter(array, filter).status()); - } -} - -static void FilterString(benchmark::State& state) { - RegressionArgs args(state); - - int32_t string_min_length = 0, string_max_length = 128; - int32_t string_mean_length = (string_max_length + string_min_length) / 2; - // for an array of 50% null strings, we need to generate twice as many strings - // to ensure that they have an average of args.size total characters - auto array_size = - static_cast(args.size / string_mean_length / (1 - args.null_proportion)); - - auto rand = random::RandomArrayGenerator(kSeed); - auto array = std::static_pointer_cast(rand.String( - array_size, string_min_length, string_max_length, args.null_proportion)); - auto filter = std::static_pointer_cast( - rand.Boolean(array_size, 0.75, args.null_proportion)); - - for (auto _ : state) { - ABORT_NOT_OK(Filter(array, filter).status()); - } -} - -BENCHMARK(FilterInt64) - ->Apply(RegressionSetArgs) - ->Args({1 << 20, 1}) - ->Args({1 << 23, 1}) - ->MinTime(1.0) - ->Unit(benchmark::TimeUnit::kNanosecond); - -BENCHMARK(FilterFixedSizeList1Int64) - ->Apply(RegressionSetArgs) - ->Args({1 << 20, 1}) - ->Args({1 << 23, 1}) - ->MinTime(1.0) - ->Unit(benchmark::TimeUnit::kNanosecond); - -BENCHMARK(FilterString) - ->Apply(RegressionSetArgs) - ->Args({1 << 20, 1}) - ->Args({1 << 23, 1}) - ->MinTime(1.0) - ->Unit(benchmark::TimeUnit::kNanosecond); - -} // namespace compute -} // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc b/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc new file mode 100644 index 00000000000..ced683833a9 --- /dev/null +++ b/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc @@ -0,0 +1,270 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "benchmark/benchmark.h" + +#include "arrow/compute/api_vector.h" +#include "arrow/compute/benchmark_util.h" +#include "arrow/compute/kernels/test_util.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/testing/random.h" + +namespace arrow { +namespace compute { + +constexpr auto kSeed = 0x0ff1ce; + +// RAII struct to handle some of the boilerplate in filter +struct FilterArgs { + // size of memory tested (per iteration) in bytes + const int64_t size; + + // proportion of nulls in the values array + const double values_null_proportion; + + // proportion of true in filter + const double selected_proportion; + + // proportion of nulls in the filter + const double filter_null_proportion; + + explicit FilterArgs(benchmark::State& state, bool filter_has_nulls) + : size(state.range(0)), + values_null_proportion(1. / static_cast(state.range(1))), + selected_proportion(1 - 1. / static_cast(state.range(1))), + filter_null_proportion(filter_has_nulls ? 1. / static_cast(state.range(1)) + : 0), + state_(state) {} + + ~FilterArgs() { + state_.counters["size"] = static_cast(size); + state_.counters["selected_pct"] = selected_proportion * 100; + state_.SetBytesProcessed(state_.iterations() * size); + } + + private: + benchmark::State& state_; +}; + +struct TakeBenchmark { + benchmark::State& state; + RegressionArgs args; + random::RandomArrayGenerator rand; + bool indices_have_nulls; + bool monotonic_indices = false; + + TakeBenchmark(benchmark::State& state, bool indices_have_nulls, + bool monotonic_indices = false) + : state(state), + args(state), + rand(kSeed), + indices_have_nulls(indices_have_nulls), + monotonic_indices(false) {} + + void Int64() { + const int64_t array_size = args.size / sizeof(int64_t); + auto values = rand.Int64(array_size, -100, 100, args.null_proportion); + Bench(values); + } + + void FSLInt64() { + const int64_t array_size = args.size / sizeof(int64_t); + auto int_array = rand.Int64(array_size, -100, 100, args.null_proportion); + auto values = std::make_shared( + fixed_size_list(int64(), 1), array_size, int_array, int_array->null_bitmap(), + int_array->null_count()); + Bench(values); + } + + void String() { + int32_t string_min_length = 0, string_max_length = 128; + int32_t string_mean_length = (string_max_length + string_min_length) / 2; + // for an array of 50% null strings, we need to generate twice as many strings + // to ensure that they have an average of args.size total characters + int64_t array_size = args.size; + if (args.null_proportion < 1) { + array_size = static_cast(args.size / string_mean_length / + (1 - args.null_proportion)); + } + auto values = std::static_pointer_cast(rand.String( + array_size, string_min_length, string_max_length, args.null_proportion)); + Bench(values); + } + + void Bench(const std::shared_ptr& values) { + bool indices_null_proportion = indices_have_nulls ? args.null_proportion : 0; + auto indices = + rand.Int32(static_cast(values->length()), 0, + static_cast(values->length() - 1), indices_null_proportion); + + if (monotonic_indices) { + auto arg_sorter = *SortToIndices(*indices); + indices = *Take(*indices, *arg_sorter); + } + + for (auto _ : state) { + ABORT_NOT_OK(Take(values, indices).status()); + } + } +}; + +struct FilterBenchmark { + benchmark::State& state; + FilterArgs args; + random::RandomArrayGenerator rand; + bool filter_has_nulls; + + FilterBenchmark(benchmark::State& state, bool filter_has_nulls) + : state(state), + args(state, filter_has_nulls), + rand(kSeed), + filter_has_nulls(filter_has_nulls) {} + + void Int64() { + const int64_t array_size = args.size / sizeof(int64_t); + auto values = std::static_pointer_cast>( + rand.Int64(array_size, -100, 100, args.values_null_proportion)); + Bench(values); + } + + void FSLInt64() { + const int64_t array_size = args.size / sizeof(int64_t); + auto int_array = std::static_pointer_cast>( + rand.Int64(array_size, -100, 100, args.values_null_proportion)); + auto values = std::make_shared( + fixed_size_list(int64(), 1), array_size, int_array, int_array->null_bitmap(), + int_array->null_count()); + Bench(values); + } + + void String() { + int32_t string_min_length = 0, string_max_length = 128; + int32_t string_mean_length = (string_max_length + string_min_length) / 2; + // for an array of 50% null strings, we need to generate twice as many strings + // to ensure that they have an average of args.size total characters + int64_t array_size = args.size; + if (args.values_null_proportion < 1) { + array_size = static_cast(args.size / string_mean_length / + (1 - args.values_null_proportion)); + } + auto values = std::static_pointer_cast(rand.String( + array_size, string_min_length, string_max_length, args.values_null_proportion)); + Bench(values); + } + + void Bench(const std::shared_ptr& values) { + auto filter = rand.Boolean(values->length(), args.selected_proportion, + args.filter_null_proportion); + for (auto _ : state) { + ABORT_NOT_OK(Filter(values, filter).status()); + } + } +}; + +static void FilterInt64FilterNoNulls(benchmark::State& state) { + return FilterBenchmark(state, false).Int64(); +} + +static void FilterInt64FilterWithNulls(benchmark::State& state) { + return FilterBenchmark(state, true).Int64(); +} + +static void FilterFSLInt64FilterNoNulls(benchmark::State& state) { + return FilterBenchmark(state, false).FSLInt64(); +} + +static void FilterFSLInt64FilterWithNulls(benchmark::State& state) { + return FilterBenchmark(state, true).FSLInt64(); +} + +static void FilterStringFilterNoNulls(benchmark::State& state) { + return FilterBenchmark(state, false).String(); +} + +static void FilterStringFilterWithNulls(benchmark::State& state) { + return FilterBenchmark(state, true).String(); +} + +static void TakeInt64RandomIndicesNoNulls(benchmark::State& state) { + return TakeBenchmark(state, false).Int64(); +} + +static void TakeInt64RandomIndicesWithNulls(benchmark::State& state) { + return TakeBenchmark(state, true).Int64(); +} + +static void TakeInt64MonotonicIndices(benchmark::State& state) { + return TakeBenchmark(state, /*indices_with_nulls=*/false, /*monotonic=*/true).Int64(); +} + +static void TakeFSLInt64RandomIndicesNoNulls(benchmark::State& state) { + return TakeBenchmark(state, false).FSLInt64(); +} + +static void TakeFSLInt64RandomIndicesWithNulls(benchmark::State& state) { + return TakeBenchmark(state, true).FSLInt64(); +} + +static void TakeFSLInt64MonotonicIndices(benchmark::State& state) { + return TakeBenchmark(state, /*indices_with_nulls=*/false, /*monotonic=*/true) + .FSLInt64(); +} + +static void TakeStringRandomIndicesNoNulls(benchmark::State& state) { + return TakeBenchmark(state, false).String(); +} + +static void TakeStringRandomIndicesWithNulls(benchmark::State& state) { + return TakeBenchmark(state, true).String(); +} + +static void TakeStringMonotonicIndices(benchmark::State& state) { + return TakeBenchmark(state, /*indices_with_nulls=*/false, /*monotonic=*/true) + .FSLInt64(); +} + +void SelectionSetArgs(benchmark::internal::Benchmark* bench) { + std::vector sizes = {kL1Size, 1 << 20}; + for (int64_t size : sizes) { + for (auto nulls : std::vector({1000, 100, 50, 10, 1})) { + bench->Args({static_cast(size), nulls}); + } + } +} + +#define SELECTION_BENCHMARK_ARGS(WHAT) WHAT->Apply(SelectionSetArgs) + +SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterInt64FilterNoNulls)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterInt64FilterWithNulls)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterFSLInt64FilterNoNulls)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterFSLInt64FilterWithNulls)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterStringFilterNoNulls)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterStringFilterWithNulls)); + +SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeInt64RandomIndicesNoNulls)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeInt64RandomIndicesWithNulls)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeFSLInt64RandomIndicesNoNulls)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeFSLInt64RandomIndicesWithNulls)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeStringRandomIndicesNoNulls)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeStringRandomIndicesWithNulls)); + +SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeInt64MonotonicIndices)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeFSLInt64MonotonicIndices)); +SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeStringMonotonicIndices)); + +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/vector_take_benchmark.cc b/cpp/src/arrow/compute/kernels/vector_take_benchmark.cc deleted file mode 100644 index 184eed31e8f..00000000000 --- a/cpp/src/arrow/compute/kernels/vector_take_benchmark.cc +++ /dev/null @@ -1,146 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "benchmark/benchmark.h" - -#include "arrow/compute/api.h" - -#include "arrow/compute/benchmark_util.h" -#include "arrow/compute/kernels/test_util.h" -#include "arrow/testing/gtest_util.h" -#include "arrow/testing/random.h" - -namespace arrow { -namespace compute { - -constexpr auto kSeed = 0x0ff1ce; - -static void TakeBenchmark(benchmark::State& state, const std::shared_ptr& values, - const std::shared_ptr& indices) { - for (auto _ : state) { - ABORT_NOT_OK(Take(values, indices).status()); - } -} - -static void TakeInt64(benchmark::State& state) { - RegressionArgs args(state); - - const int64_t array_size = args.size / sizeof(int64_t); - auto rand = random::RandomArrayGenerator(kSeed); - - auto values = rand.Int64(array_size, -100, 100, args.null_proportion); - - auto indices = rand.Int32(static_cast(array_size), 0, - static_cast(array_size - 1), args.null_proportion); - - TakeBenchmark(state, values, indices); -} - -static void TakeFixedSizeList1Int64(benchmark::State& state) { - RegressionArgs args(state); - - const int64_t array_size = args.size / sizeof(int64_t); - auto rand = random::RandomArrayGenerator(kSeed); - - auto int_array = rand.Int64(array_size, -100, 100, args.null_proportion); - auto values = std::make_shared( - fixed_size_list(int64(), 1), array_size, int_array, int_array->null_bitmap(), - int_array->null_count()); - - auto indices = rand.Int32(static_cast(array_size), 0, - static_cast(array_size - 1), args.null_proportion); - - TakeBenchmark(state, values, indices); -} - -static void TakeInt64VsFilter(benchmark::State& state) { - RegressionArgs args(state); - - const int64_t array_size = args.size / sizeof(int64_t); - auto rand = random::RandomArrayGenerator(kSeed); - - auto values = rand.Int64(array_size, -100, 100, args.null_proportion); - - auto filter = std::static_pointer_cast( - rand.Boolean(array_size, 0.75, args.null_proportion)); - - Int32Builder indices_builder; - ABORT_NOT_OK(indices_builder.Resize(array_size)); - - for (int64_t i = 0; i < array_size; ++i) { - if (filter->IsNull(i)) { - indices_builder.UnsafeAppendNull(); - } else if (filter->Value(i)) { - indices_builder.UnsafeAppend(static_cast(i)); - } - } - - std::shared_ptr indices; - ABORT_NOT_OK(indices_builder.Finish(&indices)); - TakeBenchmark(state, values, indices); -} - -static void TakeString(benchmark::State& state) { - RegressionArgs args(state); - - int32_t string_min_length = 0, string_max_length = 128; - int32_t string_mean_length = (string_max_length + string_min_length) / 2; - // for an array of 50% null strings, we need to generate twice as many strings - // to ensure that they have an average of args.size total characters - auto array_size = - static_cast(args.size / string_mean_length / (1 - args.null_proportion)); - - auto rand = random::RandomArrayGenerator(kSeed); - auto values = std::static_pointer_cast(rand.String( - array_size, string_min_length, string_max_length, args.null_proportion)); - - auto indices = rand.Int32(static_cast(array_size), 0, - static_cast(array_size - 1), args.null_proportion); - - TakeBenchmark(state, values, indices); -} - -BENCHMARK(TakeInt64) - ->Apply(RegressionSetArgs) - ->Args({1 << 20, 1}) - ->Args({1 << 23, 1}) - ->MinTime(1.0) - ->Unit(benchmark::TimeUnit::kNanosecond); - -BENCHMARK(TakeFixedSizeList1Int64) - ->Apply(RegressionSetArgs) - ->Args({1 << 20, 1}) - ->Args({1 << 23, 1}) - ->MinTime(1.0) - ->Unit(benchmark::TimeUnit::kNanosecond); - -BENCHMARK(TakeInt64VsFilter) - ->Apply(RegressionSetArgs) - ->Args({1 << 20, 1}) - ->Args({1 << 23, 1}) - ->MinTime(1.0) - ->Unit(benchmark::TimeUnit::kNanosecond); - -BENCHMARK(TakeString) - ->Apply(RegressionSetArgs) - ->Args({1 << 20, 1}) - ->Args({1 << 23, 1}) - ->MinTime(1.0) - ->Unit(benchmark::TimeUnit::kNanosecond); - -} // namespace compute -} // namespace arrow From 6e1dd3868cff1c616bf11ac07a08ee44978d2cf8 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 5 Jun 2020 12:54:15 -0500 Subject: [PATCH 2/4] Benchmark more comprehensively --- cpp/src/arrow/compute/benchmark_util.h | 2 +- .../kernels/vector_selection_benchmark.cc | 97 +++++++++++-------- 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/cpp/src/arrow/compute/benchmark_util.h b/cpp/src/arrow/compute/benchmark_util.h index d72484580df..c6aea5704e8 100644 --- a/cpp/src/arrow/compute/benchmark_util.h +++ b/cpp/src/arrow/compute/benchmark_util.h @@ -81,7 +81,7 @@ struct RegressionArgs { explicit RegressionArgs(benchmark::State& state) : size(state.range(0)), - null_proportion(1. / static_cast(state.range(1))), + null_proportion(std::min(1., 1. / static_cast(state.range(1)))), state_(state) {} ~RegressionArgs() { diff --git a/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc b/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc index ced683833a9..14de3528d6e 100644 --- a/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc +++ b/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc @@ -28,11 +28,7 @@ namespace compute { constexpr auto kSeed = 0x0ff1ce; -// RAII struct to handle some of the boilerplate in filter -struct FilterArgs { - // size of memory tested (per iteration) in bytes - const int64_t size; - +struct FilterParams { // proportion of nulls in the values array const double values_null_proportion; @@ -41,18 +37,39 @@ struct FilterArgs { // proportion of nulls in the filter const double filter_null_proportion; +}; + +std::vector g_data_sizes = {kL1Size, 1 << 20}; + +// The benchmark state parameter references this vector of cases. Test high and +// low selectivity filters. +std::vector g_filter_params = { + {0., 0.95, 0.05}, {0., 0.10, 0.05}, {0.001, 0.95, 0.05}, {0.001, 0.10, 0.05}, + {0.01, 0.95, 0.05}, {0.01, 0.10, 0.05}, {0.1, 0.95, 0.05}, {0.1, 0.10, 0.05}, + {0.9, 0.95, 0.05}, {0.9, 0.10, 0.05}}; + +// RAII struct to handle some of the boilerplate in filter +struct FilterArgs { + // size of memory tested (per iteration) in bytes + const int64_t size; - explicit FilterArgs(benchmark::State& state, bool filter_has_nulls) - : size(state.range(0)), - values_null_proportion(1. / static_cast(state.range(1))), - selected_proportion(1 - 1. / static_cast(state.range(1))), - filter_null_proportion(filter_has_nulls ? 1. / static_cast(state.range(1)) - : 0), - state_(state) {} + double values_null_proportion = 0.; + double selected_proportion = 0.; + double filter_null_proportion = 0.; + + FilterArgs(benchmark::State& state, bool filter_has_nulls) + : size(state.range(0)), state_(state) { + auto params = g_filter_params[state.range(1)]; + values_null_proportion = params.values_null_proportion; + selected_proportion = params.selected_proportion; + filter_null_proportion = filter_has_nulls ? params.filter_null_proportion : 0; + } ~FilterArgs() { state_.counters["size"] = static_cast(size); - state_.counters["selected_pct"] = selected_proportion * 100; + state_.counters["select%"] = selected_proportion * 100; + state_.counters["data null%"] = values_null_proportion * 100; + state_.counters["mask null%"] = filter_null_proportion * 100; state_.SetBytesProcessed(state_.iterations() * size); } @@ -91,7 +108,7 @@ struct TakeBenchmark { } void String() { - int32_t string_min_length = 0, string_max_length = 128; + int32_t string_min_length = 0, string_max_length = 16; int32_t string_mean_length = (string_max_length + string_min_length) / 2; // for an array of 50% null strings, we need to generate twice as many strings // to ensure that they have an average of args.size total characters @@ -152,7 +169,7 @@ struct FilterBenchmark { } void String() { - int32_t string_min_length = 0, string_max_length = 128; + int32_t string_min_length = 0, string_max_length = 16; int32_t string_mean_length = (string_max_length + string_min_length) / 2; // for an array of 50% null strings, we need to generate twice as many strings // to ensure that they have an average of args.size total characters @@ -237,34 +254,38 @@ static void TakeStringMonotonicIndices(benchmark::State& state) { .FSLInt64(); } -void SelectionSetArgs(benchmark::internal::Benchmark* bench) { - std::vector sizes = {kL1Size, 1 << 20}; - for (int64_t size : sizes) { - for (auto nulls : std::vector({1000, 100, 50, 10, 1})) { +void FilterSetArgs(benchmark::internal::Benchmark* bench) { + for (int64_t size : g_data_sizes) { + for (int i = 0; i < static_cast(g_filter_params.size()); ++i) { + bench->Args({static_cast(size), i}); + } + } +} + +BENCHMARK(FilterInt64FilterNoNulls)->Apply(FilterSetArgs); +BENCHMARK(FilterInt64FilterWithNulls)->Apply(FilterSetArgs); +BENCHMARK(FilterFSLInt64FilterNoNulls)->Apply(FilterSetArgs); +BENCHMARK(FilterFSLInt64FilterWithNulls)->Apply(FilterSetArgs); +BENCHMARK(FilterStringFilterNoNulls)->Apply(FilterSetArgs); +BENCHMARK(FilterStringFilterWithNulls)->Apply(FilterSetArgs); + +void TakeSetArgs(benchmark::internal::Benchmark* bench) { + for (int64_t size : g_data_sizes) { + for (auto nulls : std::vector({1000, 100, 50, 10, 1, 0})) { bench->Args({static_cast(size), nulls}); } } } -#define SELECTION_BENCHMARK_ARGS(WHAT) WHAT->Apply(SelectionSetArgs) - -SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterInt64FilterNoNulls)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterInt64FilterWithNulls)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterFSLInt64FilterNoNulls)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterFSLInt64FilterWithNulls)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterStringFilterNoNulls)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(FilterStringFilterWithNulls)); - -SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeInt64RandomIndicesNoNulls)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeInt64RandomIndicesWithNulls)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeFSLInt64RandomIndicesNoNulls)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeFSLInt64RandomIndicesWithNulls)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeStringRandomIndicesNoNulls)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeStringRandomIndicesWithNulls)); - -SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeInt64MonotonicIndices)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeFSLInt64MonotonicIndices)); -SELECTION_BENCHMARK_ARGS(BENCHMARK(TakeStringMonotonicIndices)); +BENCHMARK(TakeInt64RandomIndicesNoNulls)->Apply(TakeSetArgs); +BENCHMARK(TakeInt64RandomIndicesWithNulls)->Apply(TakeSetArgs); +BENCHMARK(TakeFSLInt64RandomIndicesNoNulls)->Apply(TakeSetArgs); +BENCHMARK(TakeFSLInt64RandomIndicesWithNulls)->Apply(TakeSetArgs); +BENCHMARK(TakeStringRandomIndicesNoNulls)->Apply(TakeSetArgs); +BENCHMARK(TakeStringRandomIndicesWithNulls)->Apply(TakeSetArgs); +BENCHMARK(TakeInt64MonotonicIndices)->Apply(TakeSetArgs); +BENCHMARK(TakeFSLInt64MonotonicIndices)->Apply(TakeSetArgs); +BENCHMARK(TakeStringMonotonicIndices)->Apply(TakeSetArgs); } // namespace compute } // namespace arrow From c85281dca648874789d42f276e87776ba858959f Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 5 Jun 2020 12:55:17 -0500 Subject: [PATCH 3/4] Use 16 average string size --- cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc b/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc index 14de3528d6e..e57bfcaa2e7 100644 --- a/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc +++ b/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc @@ -108,7 +108,7 @@ struct TakeBenchmark { } void String() { - int32_t string_min_length = 0, string_max_length = 16; + int32_t string_min_length = 0, string_max_length = 32; int32_t string_mean_length = (string_max_length + string_min_length) / 2; // for an array of 50% null strings, we need to generate twice as many strings // to ensure that they have an average of args.size total characters @@ -169,7 +169,7 @@ struct FilterBenchmark { } void String() { - int32_t string_min_length = 0, string_max_length = 16; + int32_t string_min_length = 0, string_max_length = 32; int32_t string_mean_length = (string_max_length + string_min_length) / 2; // for an array of 50% null strings, we need to generate twice as many strings // to ensure that they have an average of args.size total characters From bac96f8d0257488b0c76314db408b82ac04dde53 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 5 Jun 2020 13:49:09 -0500 Subject: [PATCH 4/4] Remove superfluous returns --- .../kernels/vector_selection_benchmark.cc | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc b/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc index e57bfcaa2e7..6031a907dee 100644 --- a/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc +++ b/cpp/src/arrow/compute/kernels/vector_selection_benchmark.cc @@ -193,65 +193,63 @@ struct FilterBenchmark { }; static void FilterInt64FilterNoNulls(benchmark::State& state) { - return FilterBenchmark(state, false).Int64(); + FilterBenchmark(state, false).Int64(); } static void FilterInt64FilterWithNulls(benchmark::State& state) { - return FilterBenchmark(state, true).Int64(); + FilterBenchmark(state, true).Int64(); } static void FilterFSLInt64FilterNoNulls(benchmark::State& state) { - return FilterBenchmark(state, false).FSLInt64(); + FilterBenchmark(state, false).FSLInt64(); } static void FilterFSLInt64FilterWithNulls(benchmark::State& state) { - return FilterBenchmark(state, true).FSLInt64(); + FilterBenchmark(state, true).FSLInt64(); } static void FilterStringFilterNoNulls(benchmark::State& state) { - return FilterBenchmark(state, false).String(); + FilterBenchmark(state, false).String(); } static void FilterStringFilterWithNulls(benchmark::State& state) { - return FilterBenchmark(state, true).String(); + FilterBenchmark(state, true).String(); } static void TakeInt64RandomIndicesNoNulls(benchmark::State& state) { - return TakeBenchmark(state, false).Int64(); + TakeBenchmark(state, false).Int64(); } static void TakeInt64RandomIndicesWithNulls(benchmark::State& state) { - return TakeBenchmark(state, true).Int64(); + TakeBenchmark(state, true).Int64(); } static void TakeInt64MonotonicIndices(benchmark::State& state) { - return TakeBenchmark(state, /*indices_with_nulls=*/false, /*monotonic=*/true).Int64(); + TakeBenchmark(state, /*indices_with_nulls=*/false, /*monotonic=*/true).Int64(); } static void TakeFSLInt64RandomIndicesNoNulls(benchmark::State& state) { - return TakeBenchmark(state, false).FSLInt64(); + TakeBenchmark(state, false).FSLInt64(); } static void TakeFSLInt64RandomIndicesWithNulls(benchmark::State& state) { - return TakeBenchmark(state, true).FSLInt64(); + TakeBenchmark(state, true).FSLInt64(); } static void TakeFSLInt64MonotonicIndices(benchmark::State& state) { - return TakeBenchmark(state, /*indices_with_nulls=*/false, /*monotonic=*/true) - .FSLInt64(); + TakeBenchmark(state, /*indices_with_nulls=*/false, /*monotonic=*/true).FSLInt64(); } static void TakeStringRandomIndicesNoNulls(benchmark::State& state) { - return TakeBenchmark(state, false).String(); + TakeBenchmark(state, false).String(); } static void TakeStringRandomIndicesWithNulls(benchmark::State& state) { - return TakeBenchmark(state, true).String(); + TakeBenchmark(state, true).String(); } static void TakeStringMonotonicIndices(benchmark::State& state) { - return TakeBenchmark(state, /*indices_with_nulls=*/false, /*monotonic=*/true) - .FSLInt64(); + TakeBenchmark(state, /*indices_with_nulls=*/false, /*monotonic=*/true).FSLInt64(); } void FilterSetArgs(benchmark::internal::Benchmark* bench) {