Skip to content

ARROW-16166: [C++][Compute] Utilities for assembling join output #12872

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
michalursa wants to merge 2 commits into from
Closed

ARROW-16166: [C++][Compute] Utilities for assembling join output #12872

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e27a1a3
Light-weight array utilities for assembling join output
michalursa Apr 13, 2022
b14b266
Using named constants in KeyColumnArray
michalursa Apr 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cpp/src/arrow/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,7 @@ if(ARROW_COMPUTE)
compute/function.cc
compute/function_internal.cc
compute/kernel.cc
compute/light_array.cc
compute/registry.cc
compute/kernels/aggregate_basic.cc
compute/kernels/aggregate_mode.cc
Expand Down
1 change: 1 addition & 0 deletions cpp/src/arrow/compute/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ add_arrow_compute_test(internals_test
function_test.cc
exec_test.cc
kernel_test.cc
light_array_test.cc
registry_test.cc)

add_arrow_benchmark(function_benchmark PREFIX "arrow-compute")
Expand Down
25 changes: 11 additions & 14 deletions cpp/src/arrow/compute/exec/key_compare.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ void KeyCompare::NullUpdateColumnToRow(uint32_t id_col, uint32_t num_rows_to_com
const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx,
const KeyEncoder::KeyColumnArray& col,
const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows,
uint8_t* match_bytevector) {
if (!rows.has_any_nulls(ctx) && !col.data(0)) {
Expand Down Expand Up @@ -91,7 +91,7 @@ void KeyCompare::CompareBinaryColumnToRowHelper(
uint32_t offset_within_row, uint32_t first_row_to_compare,
uint32_t num_rows_to_compare, const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map, KeyEncoder::KeyEncoderContext* ctx,
const KeyEncoder::KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
const KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
uint8_t* match_bytevector, COMPARE_FN compare_fn) {
bool is_fixed_length = rows.metadata().is_fixed_length;
if (is_fixed_length) {
Expand Down Expand Up @@ -121,7 +121,7 @@ template <bool use_selection>
void KeyCompare::CompareBinaryColumnToRow(
uint32_t offset_within_row, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector) {
uint32_t num_processed = 0;
#if defined(ARROW_HAVE_AVX2)
Expand Down Expand Up @@ -231,7 +231,7 @@ template <bool use_selection, bool is_first_varbinary_col>
void KeyCompare::CompareVarBinaryColumnToRow(
uint32_t id_varbinary_col, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector) {
#if defined(ARROW_HAVE_AVX2)
if (ctx->has_avx2()) {
Expand Down Expand Up @@ -306,14 +306,11 @@ void KeyCompare::AndByteVectors(KeyEncoder::KeyEncoderContext* ctx, uint32_t num
}
}

void KeyCompare::CompareColumnsToRows(uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx,
uint32_t* out_num_rows,
uint16_t* out_sel_left_maybe_same,
const std::vector<KeyEncoder::KeyColumnArray>& cols,
const KeyEncoder::KeyRowArray& rows) {
void KeyCompare::CompareColumnsToRows(
uint32_t num_rows_to_compare, const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map, KeyEncoder::KeyEncoderContext* ctx,
uint32_t* out_num_rows, uint16_t* out_sel_left_maybe_same,
const std::vector<KeyColumnArray>& cols, const KeyEncoder::KeyRowArray& rows) {
if (num_rows_to_compare == 0) {
*out_num_rows = 0;
return;
Expand All @@ -333,7 +330,7 @@ void KeyCompare::CompareColumnsToRows(uint32_t num_rows_to_compare,

bool is_first_column = true;
for (size_t icol = 0; icol < cols.size(); ++icol) {
const KeyEncoder::KeyColumnArray& col = cols[icol];
const KeyColumnArray& col = cols[icol];
if (col.metadata().is_null_type) {
// If this null type col is the first column, the match_bytevector_A needs to be
// initialized with 0xFF. Otherwise, the calculation can be skipped
Expand Down Expand Up @@ -374,7 +371,7 @@ void KeyCompare::CompareColumnsToRows(uint32_t num_rows_to_compare,

uint32_t ivarbinary = 0;
for (size_t icol = 0; icol < cols.size(); ++icol) {
const KeyEncoder::KeyColumnArray& col = cols[icol];
const KeyColumnArray& col = cols[icol];
if (!col.metadata().is_fixed_length) {
// Process varbinary and nulls
if (sel_left_maybe_null) {
Expand Down
35 changes: 16 additions & 19 deletions cpp/src/arrow/compute/exec/key_compare.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,22 +33,19 @@ class KeyCompare {
// Returns a single 16-bit selection vector of rows that failed comparison.
// If there is input selection on the left, the resulting selection is a filtered image
// of input selection.
static void CompareColumnsToRows(uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx,
uint32_t* out_num_rows,
uint16_t* out_sel_left_maybe_same,
const std::vector<KeyEncoder::KeyColumnArray>& cols,
const KeyEncoder::KeyRowArray& rows);
static void CompareColumnsToRows(
uint32_t num_rows_to_compare, const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map, KeyEncoder::KeyEncoderContext* ctx,
uint32_t* out_num_rows, uint16_t* out_sel_left_maybe_same,
const std::vector<KeyColumnArray>& cols, const KeyEncoder::KeyRowArray& rows);

private:
template <bool use_selection>
static void NullUpdateColumnToRow(uint32_t id_col, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx,
const KeyEncoder::KeyColumnArray& col,
const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows,
uint8_t* match_bytevector);

Expand All @@ -57,21 +54,21 @@ class KeyCompare {
uint32_t offset_within_row, uint32_t first_row_to_compare,
uint32_t num_rows_to_compare, const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map, KeyEncoder::KeyEncoderContext* ctx,
const KeyEncoder::KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
const KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
uint8_t* match_bytevector, COMPARE_FN compare_fn);

template <bool use_selection>
static void CompareBinaryColumnToRow(
uint32_t offset_within_row, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector);

template <bool use_selection, bool is_first_varbinary_col>
static void CompareVarBinaryColumnToRow(
uint32_t id_varlen_col, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector);

static void AndByteVectors(KeyEncoder::KeyEncoderContext* ctx, uint32_t num_elements,
Expand All @@ -83,29 +80,29 @@ class KeyCompare {
static uint32_t NullUpdateColumnToRowImp_avx2(
uint32_t id_col, uint32_t num_rows_to_compare, const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map, KeyEncoder::KeyEncoderContext* ctx,
const KeyEncoder::KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
const KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
uint8_t* match_bytevector);

template <bool use_selection, class COMPARE8_FN>
static uint32_t CompareBinaryColumnToRowHelper_avx2(
uint32_t offset_within_row, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector,
COMPARE8_FN compare8_fn);

template <bool use_selection>
static uint32_t CompareBinaryColumnToRowImp_avx2(
uint32_t offset_within_row, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector);

template <bool use_selection, bool is_first_varbinary_col>
static void CompareVarBinaryColumnToRowImp_avx2(
uint32_t id_varlen_col, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector);

static uint32_t AndByteVectors_avx2(uint32_t num_elements, uint8_t* bytevector_A,
Expand All @@ -114,20 +111,20 @@ class KeyCompare {
static uint32_t NullUpdateColumnToRow_avx2(
bool use_selection, uint32_t id_col, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector);

static uint32_t CompareBinaryColumnToRow_avx2(
bool use_selection, uint32_t offset_within_row, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector);

static void CompareVarBinaryColumnToRow_avx2(
bool use_selection, bool is_first_varbinary_col, uint32_t id_varlen_col,
uint32_t num_rows_to_compare, const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map, KeyEncoder::KeyEncoderContext* ctx,
const KeyEncoder::KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
const KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
uint8_t* match_bytevector);

#endif
Expand Down
14 changes: 7 additions & 7 deletions cpp/src/arrow/compute/exec/key_compare_avx2.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ template <bool use_selection>
uint32_t KeyCompare::NullUpdateColumnToRowImp_avx2(
uint32_t id_col, uint32_t num_rows_to_compare, const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map, KeyEncoder::KeyEncoderContext* ctx,
const KeyEncoder::KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
const KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
uint8_t* match_bytevector) {
if (!rows.has_any_nulls(ctx) && !col.data(0)) {
return num_rows_to_compare;
Expand Down Expand Up @@ -180,7 +180,7 @@ template <bool use_selection, class COMPARE8_FN>
uint32_t KeyCompare::CompareBinaryColumnToRowHelper_avx2(
uint32_t offset_within_row, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector,
COMPARE8_FN compare8_fn) {
bool is_fixed_length = rows.metadata().is_fixed_length;
Expand Down Expand Up @@ -419,7 +419,7 @@ template <bool use_selection>
uint32_t KeyCompare::CompareBinaryColumnToRowImp_avx2(
uint32_t offset_within_row, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector) {
uint32_t col_width = col.metadata().fixed_length;
if (col_width == 0) {
Expand Down Expand Up @@ -503,7 +503,7 @@ template <bool use_selection, bool is_first_varbinary_col>
void KeyCompare::CompareVarBinaryColumnToRowImp_avx2(
uint32_t id_varbinary_col, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector) {
const uint32_t* offsets_left = col.offsets();
const uint32_t* offsets_right = rows.offsets();
Expand Down Expand Up @@ -569,7 +569,7 @@ uint32_t KeyCompare::AndByteVectors_avx2(uint32_t num_elements, uint8_t* bytevec
uint32_t KeyCompare::NullUpdateColumnToRow_avx2(
bool use_selection, uint32_t id_col, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector) {
if (use_selection) {
return NullUpdateColumnToRowImp_avx2<true>(id_col, num_rows_to_compare,
Expand All @@ -585,7 +585,7 @@ uint32_t KeyCompare::NullUpdateColumnToRow_avx2(
uint32_t KeyCompare::CompareBinaryColumnToRow_avx2(
bool use_selection, uint32_t offset_within_row, uint32_t num_rows_to_compare,
const uint16_t* sel_left_maybe_null, const uint32_t* left_to_right_map,
KeyEncoder::KeyEncoderContext* ctx, const KeyEncoder::KeyColumnArray& col,
KeyEncoder::KeyEncoderContext* ctx, const KeyColumnArray& col,
const KeyEncoder::KeyRowArray& rows, uint8_t* match_bytevector) {
if (use_selection) {
return CompareBinaryColumnToRowImp_avx2<true>(offset_within_row, num_rows_to_compare,
Expand All @@ -602,7 +602,7 @@ void KeyCompare::CompareVarBinaryColumnToRow_avx2(
bool use_selection, bool is_first_varbinary_col, uint32_t id_varlen_col,
uint32_t num_rows_to_compare, const uint16_t* sel_left_maybe_null,
const uint32_t* left_to_right_map, KeyEncoder::KeyEncoderContext* ctx,
const KeyEncoder::KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
const KeyColumnArray& col, const KeyEncoder::KeyRowArray& rows,
uint8_t* match_bytevector) {
if (use_selection) {
if (is_first_varbinary_col) {
Expand Down
Loading