Skip to content

ARROW-11125: [Rust] Logical equality for list arrays #9093

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
nevi-me wants to merge 6 commits into from
Closed

ARROW-11125: [Rust] Logical equality for list arrays #9093

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
10be8bb
ARROW-11125: [Rust] Logical equality for list arrays
nevi-me Jan 4, 2021
4c4ac46
update renamed functions
nevi-me Jan 4, 2021
a2941dd
ignore clippy warning
nevi-me Jan 4, 2021
66bb98e
fix integration test failures
nevi-me Jan 5, 2021
93017a2
address review comments
nevi-me Jan 5, 2021
01b28b5
take borrowed reference of buffer
nevi-me Jan 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions rust/arrow/src/array/data.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ pub(crate) fn count_nulls(
offset: usize,
len: usize,
) -> usize {
if let Some(ref buf) = null_bit_buffer {
if let Some(buf) = null_bit_buffer {
len.checked_sub(buf.count_set_bits_offset(offset, len))
.unwrap()
} else {
Expand Down Expand Up @@ -337,7 +337,6 @@ mod tests {

use std::sync::Arc;

use crate::buffer::Buffer;
use crate::datatypes::ToByteSlice;
use crate::util::bit_util;

Expand Down
61 changes: 43 additions & 18 deletions rust/arrow/src/array/equal/boolean.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,35 +15,60 @@
// specific language governing permissions and limitations
// under the License.

use crate::array::ArrayData;
use crate::array::{data::count_nulls, ArrayData};
use crate::buffer::Buffer;
use crate::util::bit_util::get_bit;

use super::utils::equal_bits;

pub(super) fn boolean_equal(
lhs: &ArrayData,
rhs: &ArrayData,
lhs_nulls: Option<&Buffer>,
rhs_nulls: Option<&Buffer>,
lhs_start: usize,
rhs_start: usize,
len: usize,
) -> bool {
let lhs_values = lhs.buffers()[0].as_slice();
let rhs_values = rhs.buffers()[0].as_slice();

// TODO: we can do this more efficiently if all values are not-null
(0..len).all(|i| {
let lhs_pos = lhs_start + i;
let rhs_pos = rhs_start + i;
let lhs_is_null = lhs.is_null(lhs_pos);
let rhs_is_null = rhs.is_null(rhs_pos);

lhs_is_null
|| (lhs_is_null == rhs_is_null)
&& equal_bits(
lhs_values,
rhs_values,
lhs_pos + lhs.offset(),
rhs_pos + rhs.offset(),
1,
)
})
let lhs_null_count = count_nulls(lhs_nulls, lhs_start, len);
let rhs_null_count = count_nulls(rhs_nulls, rhs_start, len);

if lhs_null_count == 0 && rhs_null_count == 0 {
(0..len).all(|i| {
let lhs_pos = lhs_start + i;
let rhs_pos = rhs_start + i;

equal_bits(
lhs_values,
rhs_values,
lhs_pos + lhs.offset(),
rhs_pos + rhs.offset(),
1,
)
})
} else {
// get a ref of the null buffer bytes, to use in testing for nullness
let lhs_null_bytes = lhs_nulls.as_ref().unwrap().as_slice();
Copy link
Contributor

@alamb alamb Jan 5, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible for lhs_nulls == Some(..) but rhs_nulls == None (and visa versa?) Given they are optional arguments I wasn't sure if they would always both be either None or Some

let rhs_null_bytes = rhs_nulls.as_ref().unwrap().as_slice();

(0..len).all(|i| {
let lhs_pos = lhs_start + i;
let rhs_pos = rhs_start + i;
let lhs_is_null = !get_bit(lhs_null_bytes, lhs_pos);
let rhs_is_null = !get_bit(rhs_null_bytes, rhs_pos);

lhs_is_null
|| (lhs_is_null == rhs_is_null)
&& equal_bits(
lhs_values,
rhs_values,
lhs_pos + lhs.offset(),
rhs_pos + rhs.offset(),
1,
)
})
}
}
19 changes: 15 additions & 4 deletions rust/arrow/src/array/equal/decimal.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,18 @@
// specific language governing permissions and limitations
// under the License.

use crate::{array::ArrayData, datatypes::DataType};
use crate::array::{data::count_nulls, ArrayData};
use crate::buffer::Buffer;
use crate::datatypes::DataType;
use crate::util::bit_util::get_bit;

use super::utils::equal_len;

pub(super) fn decimal_equal(
lhs: &ArrayData,
rhs: &ArrayData,
lhs_nulls: Option<&Buffer>,
rhs_nulls: Option<&Buffer>,
lhs_start: usize,
rhs_start: usize,
len: usize,
Expand All @@ -34,7 +39,10 @@ pub(super) fn decimal_equal(
let lhs_values = &lhs.buffers()[0].as_slice()[lhs.offset() * size..];
let rhs_values = &rhs.buffers()[0].as_slice()[rhs.offset() * size..];

if lhs.null_count() == 0 && rhs.null_count() == 0 {
let lhs_null_count = count_nulls(lhs_nulls, lhs_start, len);
let rhs_null_count = count_nulls(rhs_nulls, rhs_start, len);

if lhs_null_count == 0 && rhs_null_count == 0 {
equal_len(
lhs_values,
rhs_values,
Expand All @@ -43,13 +51,16 @@ pub(super) fn decimal_equal(
size * len,
)
} else {
// get a ref of the null buffer bytes, to use in testing for nullness
let lhs_null_bytes = lhs_nulls.as_ref().unwrap().as_slice();
let rhs_null_bytes = rhs_nulls.as_ref().unwrap().as_slice();
// with nulls, we need to compare item by item whenever it is not null
(0..len).all(|i| {
let lhs_pos = lhs_start + i;
let rhs_pos = rhs_start + i;

let lhs_is_null = lhs.is_null(lhs_pos);
let rhs_is_null = rhs.is_null(rhs_pos);
let lhs_is_null = !get_bit(lhs_null_bytes, lhs_pos + lhs.offset());
let rhs_is_null = !get_bit(rhs_null_bytes, rhs_pos + lhs.offset());

lhs_is_null
|| (lhs_is_null == rhs_is_null)
Expand Down
19 changes: 15 additions & 4 deletions rust/arrow/src/array/equal/dictionary.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,18 @@
// specific language governing permissions and limitations
// under the License.

use crate::{array::ArrayData, datatypes::ArrowNativeType};
use crate::array::{data::count_nulls, ArrayData};
use crate::buffer::Buffer;
use crate::datatypes::ArrowNativeType;
use crate::util::bit_util::get_bit;

use super::equal_range;

pub(super) fn dictionary_equal<T: ArrowNativeType>(
lhs: &ArrayData,
rhs: &ArrayData,
lhs_nulls: Option<&Buffer>,
rhs_nulls: Option<&Buffer>,
lhs_start: usize,
rhs_start: usize,
len: usize,
Expand All @@ -32,7 +37,10 @@ pub(super) fn dictionary_equal<T: ArrowNativeType>(
let lhs_values = lhs.child_data()[0].as_ref();
let rhs_values = rhs.child_data()[0].as_ref();

if lhs.null_count() == 0 && rhs.null_count() == 0 {
let lhs_null_count = count_nulls(lhs_nulls, lhs_start, len);
let rhs_null_count = count_nulls(rhs_nulls, rhs_start, len);

if lhs_null_count == 0 && rhs_null_count == 0 {
(0..len).all(|i| {
let lhs_pos = lhs_start + i;
let rhs_pos = rhs_start + i;
Expand All @@ -48,12 +56,15 @@ pub(super) fn dictionary_equal<T: ArrowNativeType>(
)
})
} else {
// get a ref of the null buffer bytes, to use in testing for nullness
let lhs_null_bytes = lhs_nulls.as_ref().unwrap().as_slice();
let rhs_null_bytes = rhs_nulls.as_ref().unwrap().as_slice();
(0..len).all(|i| {
let lhs_pos = lhs_start + i;
let rhs_pos = rhs_start + i;

let lhs_is_null = lhs.is_null(lhs_pos);
let rhs_is_null = rhs.is_null(rhs_pos);
let lhs_is_null = !get_bit(lhs_null_bytes, lhs_pos);
let rhs_is_null = !get_bit(rhs_null_bytes, rhs_pos);

lhs_is_null
|| (lhs_is_null == rhs_is_null)
Expand Down
19 changes: 15 additions & 4 deletions rust/arrow/src/array/equal/fixed_binary.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,18 @@
// specific language governing permissions and limitations
// under the License.

use crate::{array::ArrayData, datatypes::DataType};
use crate::array::{data::count_nulls, ArrayData};
use crate::buffer::Buffer;
use crate::datatypes::DataType;
use crate::util::bit_util::get_bit;

use super::utils::equal_len;

pub(super) fn fixed_binary_equal(
lhs: &ArrayData,
rhs: &ArrayData,
lhs_nulls: Option<&Buffer>,
rhs_nulls: Option<&Buffer>,
lhs_start: usize,
rhs_start: usize,
len: usize,
Expand All @@ -34,7 +39,10 @@ pub(super) fn fixed_binary_equal(
let lhs_values = &lhs.buffers()[0].as_slice()[lhs.offset() * size..];
let rhs_values = &rhs.buffers()[0].as_slice()[rhs.offset() * size..];

if lhs.null_count() == 0 && rhs.null_count() == 0 {
let lhs_null_count = count_nulls(lhs_nulls, lhs_start, len);
let rhs_null_count = count_nulls(rhs_nulls, rhs_start, len);

if lhs_null_count == 0 && rhs_null_count == 0 {
equal_len(
lhs_values,
rhs_values,
Expand All @@ -43,13 +51,16 @@ pub(super) fn fixed_binary_equal(
size * len,
)
} else {
// get a ref of the null buffer bytes, to use in testing for nullness
let lhs_null_bytes = lhs_nulls.as_ref().unwrap().as_slice();
let rhs_null_bytes = rhs_nulls.as_ref().unwrap().as_slice();
// with nulls, we need to compare item by item whenever it is not null
(0..len).all(|i| {
let lhs_pos = lhs_start + i;
let rhs_pos = rhs_start + i;

let lhs_is_null = lhs.is_null(lhs_pos);
let rhs_is_null = rhs.is_null(rhs_pos);
let lhs_is_null = !get_bit(lhs_null_bytes, lhs_pos);
let rhs_is_null = !get_bit(rhs_null_bytes, rhs_pos);

lhs_is_null
|| (lhs_is_null == rhs_is_null)
Expand Down
19 changes: 15 additions & 4 deletions rust/arrow/src/array/equal/fixed_list.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,18 @@
// specific language governing permissions and limitations
// under the License.

use crate::{array::ArrayData, datatypes::DataType};
use crate::array::{data::count_nulls, ArrayData};
use crate::buffer::Buffer;
use crate::datatypes::DataType;
use crate::util::bit_util::get_bit;

use super::equal_range;

pub(super) fn fixed_list_equal(
lhs: &ArrayData,
rhs: &ArrayData,
lhs_nulls: Option<&Buffer>,
rhs_nulls: Option<&Buffer>,
lhs_start: usize,
rhs_start: usize,
len: usize,
Expand All @@ -34,7 +39,10 @@ pub(super) fn fixed_list_equal(
let lhs_values = lhs.child_data()[0].as_ref();
let rhs_values = rhs.child_data()[0].as_ref();

if lhs.null_count() == 0 && rhs.null_count() == 0 {
let lhs_null_count = count_nulls(lhs_nulls, lhs_start, len);
let rhs_null_count = count_nulls(rhs_nulls, rhs_start, len);

if lhs_null_count == 0 && rhs_null_count == 0 {
equal_range(
lhs_values,
rhs_values,
Expand All @@ -45,13 +53,16 @@ pub(super) fn fixed_list_equal(
size * len,
)
} else {
// get a ref of the null buffer bytes, to use in testing for nullness
let lhs_null_bytes = lhs_nulls.as_ref().unwrap().as_slice();
let rhs_null_bytes = rhs_nulls.as_ref().unwrap().as_slice();
// with nulls, we need to compare item by item whenever it is not null
(0..len).all(|i| {
let lhs_pos = lhs_start + i;
let rhs_pos = rhs_start + i;

let lhs_is_null = lhs.is_null(lhs_pos);
let rhs_is_null = rhs.is_null(rhs_pos);
let lhs_is_null = !get_bit(lhs_null_bytes, lhs_pos);
let rhs_is_null = !get_bit(rhs_null_bytes, rhs_pos);

lhs_is_null
|| (lhs_is_null == rhs_is_null)
Expand Down
Loading