Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
91aaf46
Remove dead code in memory pool
jvanstraten Jan 10, 2022
297f0d7
Add MakeBufferOfZeros() and supporting MemoryPool abstractions
jvanstraten Jan 10, 2022
389a188
Separate MakeArrayOfNull and MakeArrayFromScalar into mutable and imm…
jvanstraten Jan 18, 2022
4db1397
Fix one instance of mutable access to result of MakeArrayOfNulls()
jvanstraten Jan 19, 2022
fc5fac9
Fix compilation on some compilers, clean up slightly
jvanstraten Jan 19, 2022
d181c1d
Fix missing pointer dereference
jvanstraten Jan 19, 2022
2d5064c
Refactor NULLPTR to nullptr in .cc files
jvanstraten Jan 19, 2022
700f5c9
Allow mmap-based optimization for immutable zeros to be disabled usin…
jvanstraten Jan 19, 2022
5dc66ef
Fix typo/lint violation
jvanstraten Jan 19, 2022
bd3810c
Fix docstrings to align with earlier refactor
jvanstraten Feb 9, 2022
498d12f
Refactor @param to \param
jvanstraten Feb 9, 2022
9d055d1
Use BytesForBits utility function
jvanstraten Feb 9, 2022
7ce1bc6
Fix MakeMutableArrayOfNull not being fully mutable for nested types
jvanstraten Feb 9, 2022
9056106
Refactor ImmutableNullArrayFactory to NullArrayFactory and NullArrayF…
jvanstraten Feb 9, 2022
81aa078
Hide ImmutableZeros class using Buffer interface, extend abstractions…
jvanstraten Feb 10, 2022
3a10fca
Fix lint violation due to typo
jvanstraten Feb 14, 2022
a4c3413
Fix possibly custom memory pool not used for creation of buffer
jvanstraten Feb 14, 2022
41ebefc
Add missing is_mutable() override
jvanstraten Feb 14, 2022
e628688
Rewrite immutable zeros allocation
jvanstraten Feb 15, 2022
cbf33de
Add TODO for using dedicated zero-initialized alloc functions of allo…
jvanstraten Feb 15, 2022
d89c297
Rename constant to satisfy naming conventions
jvanstraten Feb 15, 2022
3dda474
Remove unnecessary null check
jvanstraten Feb 15, 2022
3738819
Move OS-specific mmap logic for immutable zeros to io_util.[h|cc]
jvanstraten Feb 15, 2022
dee486a
Use internal::atomic[load|store] for shared_ptr for compatibility
jvanstraten Feb 16, 2022
976dcf6
Revert addition of special memory manager for immutable zeros
jvanstraten Feb 17, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 120 additions & 103 deletions cpp/src/arrow/array/array_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -368,25 +368,27 @@ TEST_F(TestArray, TestMakeArrayOfNull) {
// clang-format on
};

for (int64_t length : {0, 1, 16, 133}) {
for (auto type : types) {
ARROW_SCOPED_TRACE("type = ", type->ToString());
ASSERT_OK_AND_ASSIGN(auto array, MakeArrayOfNull(type, length));
ASSERT_EQ(array->type(), type);
ASSERT_OK(array->ValidateFull());
ASSERT_EQ(array->length(), length);
if (is_union(type->id())) {
// For unions, MakeArrayOfNull places the nulls in the children
ASSERT_EQ(array->null_count(), 0);
const auto& union_array = checked_cast<const UnionArray&>(*array);
for (int i = 0; i < union_array.num_fields(); ++i) {
ASSERT_EQ(union_array.field(i)->null_count(), union_array.field(i)->length());
}
} else {
ASSERT_EQ(array->null_count(), length);
for (int64_t i = 0; i < length; ++i) {
ASSERT_TRUE(array->IsNull(i));
ASSERT_FALSE(array->IsValid(i));
for (auto fun : {MakeArrayOfNull, MakeMutableArrayOfNull}) {
for (int64_t length : {0, 1, 16, 133}) {
for (auto type : types) {
ARROW_SCOPED_TRACE("type = ", type->ToString());
ASSERT_OK_AND_ASSIGN(auto array, fun(type, length, default_memory_pool()));
ASSERT_EQ(array->type(), type);
ASSERT_OK(array->ValidateFull());
ASSERT_EQ(array->length(), length);
if (is_union(type->id())) {
// For unions, MakeArrayOfNull places the nulls in the children
ASSERT_EQ(array->null_count(), 0);
const auto& union_array = checked_cast<const UnionArray&>(*array);
for (int i = 0; i < union_array.num_fields(); ++i) {
ASSERT_EQ(union_array.field(i)->null_count(), union_array.field(i)->length());
}
} else {
ASSERT_EQ(array->null_count(), length);
for (int64_t i = 0; i < length; ++i) {
ASSERT_TRUE(array->IsNull(i));
ASSERT_FALSE(array->IsValid(i));
}
}
}
}
Expand All @@ -397,51 +399,56 @@ TEST_F(TestArray, TestMakeArrayOfNullUnion) {
// Unions need special checking -- the top level null count is 0 (per
// ARROW-9222) so we check the first child to make sure is contains all nulls
// and check that the type_ids all point to the first child
const int64_t union_length = 10;
auto s_union_ty = sparse_union({field("a", utf8()), field("b", int32())}, {0, 1});
ASSERT_OK_AND_ASSIGN(auto s_union_nulls, MakeArrayOfNull(s_union_ty, union_length));
ASSERT_OK(s_union_nulls->ValidateFull());
ASSERT_EQ(s_union_nulls->null_count(), 0);
{
const auto& typed_union = checked_cast<const SparseUnionArray&>(*s_union_nulls);
ASSERT_EQ(typed_union.field(0)->null_count(), union_length);

// Check type codes are all 0
for (int i = 0; i < union_length; ++i) {
ASSERT_EQ(typed_union.raw_type_codes()[i], 0);
for (auto fun : {MakeArrayOfNull, MakeMutableArrayOfNull}) {
const int64_t union_length = 10;
auto s_union_ty = sparse_union({field("a", utf8()), field("b", int32())}, {0, 1});
ASSERT_OK_AND_ASSIGN(auto s_union_nulls,
fun(s_union_ty, union_length, default_memory_pool()));
ASSERT_OK(s_union_nulls->ValidateFull());
ASSERT_EQ(s_union_nulls->null_count(), 0);
{
const auto& typed_union = checked_cast<const SparseUnionArray&>(*s_union_nulls);
ASSERT_EQ(typed_union.field(0)->null_count(), union_length);

// Check type codes are all 0
for (int i = 0; i < union_length; ++i) {
ASSERT_EQ(typed_union.raw_type_codes()[i], 0);
}
}
}

s_union_ty = sparse_union({field("a", utf8()), field("b", int32())}, {2, 7});
ASSERT_OK_AND_ASSIGN(s_union_nulls, MakeArrayOfNull(s_union_ty, union_length));
ASSERT_OK(s_union_nulls->ValidateFull());
ASSERT_EQ(s_union_nulls->null_count(), 0);
{
const auto& typed_union = checked_cast<const SparseUnionArray&>(*s_union_nulls);
ASSERT_EQ(typed_union.field(0)->null_count(), union_length);

// Check type codes are all 2
for (int i = 0; i < union_length; ++i) {
ASSERT_EQ(typed_union.raw_type_codes()[i], 2);
s_union_ty = sparse_union({field("a", utf8()), field("b", int32())}, {2, 7});
ASSERT_OK_AND_ASSIGN(s_union_nulls,
fun(s_union_ty, union_length, default_memory_pool()));
ASSERT_OK(s_union_nulls->ValidateFull());
ASSERT_EQ(s_union_nulls->null_count(), 0);
{
const auto& typed_union = checked_cast<const SparseUnionArray&>(*s_union_nulls);
ASSERT_EQ(typed_union.field(0)->null_count(), union_length);

// Check type codes are all 2
for (int i = 0; i < union_length; ++i) {
ASSERT_EQ(typed_union.raw_type_codes()[i], 2);
}
}
}

auto d_union_ty = dense_union({field("a", utf8()), field("b", int32())}, {0, 1});
ASSERT_OK_AND_ASSIGN(auto d_union_nulls, MakeArrayOfNull(d_union_ty, union_length));
ASSERT_OK(d_union_nulls->ValidateFull());
ASSERT_EQ(d_union_nulls->null_count(), 0);
{
const auto& typed_union = checked_cast<const DenseUnionArray&>(*d_union_nulls);

// Child field has length 1 which is a null element
ASSERT_EQ(typed_union.field(0)->length(), 1);
ASSERT_EQ(typed_union.field(0)->null_count(), 1);

// Check type codes are all 0 and the offsets point to the first element of
// the first child
for (int i = 0; i < union_length; ++i) {
ASSERT_EQ(typed_union.raw_type_codes()[i], 0);
ASSERT_EQ(typed_union.raw_value_offsets()[i], 0);
auto d_union_ty = dense_union({field("a", utf8()), field("b", int32())}, {0, 1});
ASSERT_OK_AND_ASSIGN(auto d_union_nulls,
fun(d_union_ty, union_length, default_memory_pool()));
ASSERT_OK(d_union_nulls->ValidateFull());
ASSERT_EQ(d_union_nulls->null_count(), 0);
{
const auto& typed_union = checked_cast<const DenseUnionArray&>(*d_union_nulls);

// Child field has length 1 which is a null element
ASSERT_EQ(typed_union.field(0)->length(), 1);
ASSERT_EQ(typed_union.field(0)->null_count(), 1);

// Check type codes are all 0 and the offsets point to the first element of
// the first child
for (int i = 0; i < union_length; ++i) {
ASSERT_EQ(typed_union.raw_type_codes()[i], 0);
ASSERT_EQ(typed_union.raw_value_offsets()[i], 0);
}
}
}
}
Expand Down Expand Up @@ -579,44 +586,48 @@ static ScalarVector GetScalars() {
}

TEST_F(TestArray, TestMakeArrayFromScalar) {
ASSERT_OK_AND_ASSIGN(auto null_array, MakeArrayFromScalar(NullScalar(), 5));
ASSERT_OK(null_array->ValidateFull());
ASSERT_EQ(null_array->length(), 5);
ASSERT_EQ(null_array->null_count(), 5);

auto scalars = GetScalars();
for (auto fun : {MakeArrayFromScalar, MakeMutableArrayFromScalar}) {
ASSERT_OK_AND_ASSIGN(auto null_array, fun(NullScalar(), 5, default_memory_pool()));
ASSERT_OK(null_array->ValidateFull());
ASSERT_EQ(null_array->length(), 5);
ASSERT_EQ(null_array->null_count(), 5);

auto scalars = GetScalars();

for (int64_t length : {16}) {
for (auto scalar : scalars) {
ASSERT_OK_AND_ASSIGN(auto array, fun(*scalar, length, default_memory_pool()));
ASSERT_OK(array->ValidateFull());
ASSERT_EQ(array->length(), length);
ASSERT_EQ(array->null_count(), 0);

for (int64_t length : {16}) {
for (auto scalar : scalars) {
ASSERT_OK_AND_ASSIGN(auto array, MakeArrayFromScalar(*scalar, length));
ASSERT_OK(array->ValidateFull());
ASSERT_EQ(array->length(), length);
ASSERT_EQ(array->null_count(), 0);

// test case for ARROW-13321
for (int64_t i : std::vector<int64_t>{0, length / 2, length - 1}) {
ASSERT_OK_AND_ASSIGN(auto s, array->GetScalar(i));
AssertScalarsEqual(*s, *scalar, /*verbose=*/true);
// test case for ARROW-13321
for (int64_t i : std::vector<int64_t>{0, length / 2, length - 1}) {
ASSERT_OK_AND_ASSIGN(auto s, array->GetScalar(i));
AssertScalarsEqual(*s, *scalar, /*verbose=*/true);
}
}
}
}

for (auto scalar : scalars) {
AssertAppendScalar(pool_, scalar);
for (auto scalar : scalars) {
AssertAppendScalar(pool_, scalar);
}
}
}

TEST_F(TestArray, TestMakeArrayFromScalarSliced) {
// Regression test for ARROW-13437
auto scalars = GetScalars();

for (auto scalar : scalars) {
SCOPED_TRACE(scalar->type->ToString());
ASSERT_OK_AND_ASSIGN(auto array, MakeArrayFromScalar(*scalar, 32));
auto sliced = array->Slice(1, 4);
ASSERT_EQ(sliced->length(), 4);
ASSERT_EQ(sliced->null_count(), 0);
ARROW_EXPECT_OK(sliced->ValidateFull());
for (auto fun : {MakeArrayFromScalar, MakeMutableArrayFromScalar}) {
for (auto scalar : scalars) {
SCOPED_TRACE(scalar->type->ToString());
ASSERT_OK_AND_ASSIGN(auto array, fun(*scalar, 32, default_memory_pool()));
auto sliced = array->Slice(1, 4);
ASSERT_EQ(sliced->length(), 4);
ASSERT_EQ(sliced->null_count(), 0);
ARROW_EXPECT_OK(sliced->ValidateFull());
}
}
}

Expand All @@ -626,14 +637,16 @@ TEST_F(TestArray, TestMakeArrayFromDictionaryScalar) {
ASSERT_OK_AND_ASSIGN(auto value, MakeScalar(int8(), 1));
auto scalar = DictionaryScalar({value, dictionary}, type);

ASSERT_OK_AND_ASSIGN(auto array, MakeArrayFromScalar(scalar, 4));
ASSERT_OK(array->ValidateFull());
ASSERT_EQ(array->length(), 4);
ASSERT_EQ(array->null_count(), 0);
for (auto fun : {MakeArrayFromScalar, MakeMutableArrayFromScalar}) {
ASSERT_OK_AND_ASSIGN(auto array, fun(scalar, 4, default_memory_pool()));
ASSERT_OK(array->ValidateFull());
ASSERT_EQ(array->length(), 4);
ASSERT_EQ(array->null_count(), 0);

for (int i = 0; i < 4; i++) {
ASSERT_OK_AND_ASSIGN(auto item, array->GetScalar(i));
ASSERT_TRUE(item->Equals(scalar));
for (int i = 0; i < 4; i++) {
ASSERT_OK_AND_ASSIGN(auto item, array->GetScalar(i));
ASSERT_TRUE(item->Equals(scalar));
}
}
}

Expand All @@ -643,17 +656,21 @@ TEST_F(TestArray, TestMakeArrayFromMapScalar) {
R"([{"key": "a", "value": 1}, {"key": "b", "value": 2}])");
auto scalar = MapScalar(value);

ASSERT_OK_AND_ASSIGN(auto array, MakeArrayFromScalar(scalar, 11));
ASSERT_OK(array->ValidateFull());
ASSERT_EQ(array->length(), 11);
ASSERT_EQ(array->null_count(), 0);
for (auto fun : {MakeArrayFromScalar, MakeMutableArrayFromScalar}) {
ASSERT_OK_AND_ASSIGN(auto array, fun(scalar, 11, default_memory_pool()));
ASSERT_OK(array->ValidateFull());
ASSERT_EQ(array->length(), 11);
ASSERT_EQ(array->null_count(), 0);

for (int i = 0; i < 11; i++) {
ASSERT_OK_AND_ASSIGN(auto item, array->GetScalar(i));
ASSERT_TRUE(item->Equals(scalar));
}
for (int i = 0; i < 11; i++) {
ASSERT_OK_AND_ASSIGN(auto item, array->GetScalar(i));
ASSERT_TRUE(item->Equals(scalar));
}

AssertAppendScalar(pool_, std::make_shared<MapScalar>(scalar));
if (fun != MakeMutableArrayFromScalar) {
AssertAppendScalar(pool_, std::make_shared<MapScalar>(scalar));
}
}
}

TEST_F(TestArray, TestMakeEmptyArray) {
Expand Down
Loading