From 359b29e746f13f9335b355c36e812935f45f2a71 Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Tue, 21 Dec 2021 14:40:17 -0500 Subject: [PATCH 01/17] Adding validation for Date64Type, Time32Type, and Time64Type --- cpp/src/arrow/array/validate.cc | 80 +++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index 7c4ab6d5248..6954528da83 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -166,6 +166,86 @@ struct ValidateArrayImpl { return Status::OK(); } + Status Visit(const Date64Type& type) { + // check that data is divisible by 8.64e7 (= 1000ms * 60s * 60mins * 24hrs) + RETURN_NOT_OK(ValidateFixedWidthBuffers()); + + using c_type = typename Date64Type::c_type; + if (full_validation) { + c_type fullDay = 1000 * 60 * 60 * 24; + return VisitArrayDataInline( + data, + [&](c_type date) { + if(date % fullDay != 0) { + return Status::Invalid(type, date, "ms does not represent a whole number of days"); + } + return Status::OK(); + }, + []() { return Status::OK(); }); + } + return Status::OK(); + } + + Status Visit(const Time32Type& type) { + // check unit + // if unit is s => data must be within [0, 8.64e4) + // if unit is ms => data must be within [0, 8.64e7) + RETURN_NOT_OK(ValidateFixedWidthBuffers()); + + using c_type = typename Time32Type::c_type; + if (full_validation) { + c_type fullDay_s = 60 * 60 * 24; + c_type fullDay_ms = fullDay_s * 1000; + return VisitArrayDataInline( + data, + [&](c_type time) { + if(type.unit() == TimeUnit::SECOND && (time < 0 || time >= fullDay_s)) { + return Status::Invalid(type, " ", time, + "s does not fit within the acceptable range of [0, ", + fullDay_s, ") s"); + } + if(type.unit() == TimeUnit::MILLI && (time < 0 || time >= fullDay_ms)) { + return Status::Invalid(type, " ", time, + "ms does not fit within the acceptable range of [0, ", + fullDay_ms, ") ms"); + } + return Status::OK(); + }, + []() { return Status::OK(); }); + } + return Status::OK(); + } + + Status Visit(const Time64Type& type) { + // check unit + // if unit is us => data must be within [0, 8.64e10) + // if unit is ns => data must be within [0, 8.64e13) + RETURN_NOT_OK(ValidateFixedWidthBuffers()); + + using c_type = typename Time32Type::c_type; + if (full_validation) { + c_type fullDay_us = 1000000 * 60 * 60 * 24; + c_type fullDay_ns = fullDay_us * 1000; + return VisitArrayDataInline( + data, + [&](c_type time) { + if(type.unit() == TimeUnit::MICRO && (time < 0 || time >= fullDay_us)) { + return Status::Invalid(type, " ", time, + "us does not fit within the acceptable range of [0, ", + fullDay_us, ") us"); + } + if(type.unit() == TimeUnit::NANO && (time < 0 || time >= fullDay_ns)) { + return Status::Invalid(type, " ", time, + "ns does not fit within the acceptable range of [0, ", + fullDay_ns, ") ns"); + } + return Status::OK(); + }, + []() { return Status::OK(); }); + } + return Status::OK(); + } + Status Visit(const BinaryType& type) { return ValidateBinaryLike(type); } Status Visit(const LargeBinaryType& type) { return ValidateBinaryLike(type); } From b13946bb78c1cf2edbbb47814f5f8ee5fb758af9 Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Tue, 21 Dec 2021 17:37:10 -0500 Subject: [PATCH 02/17] Corrected type usage for Time64Type check --- cpp/src/arrow/array/validate.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index 433f57b1d08..2c328ffe1bc 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -222,9 +222,9 @@ struct ValidateArrayImpl { // if unit is ns => data must be within [0, 8.64e13) RETURN_NOT_OK(ValidateFixedWidthBuffers()); - using c_type = typename Time32Type::c_type; + using c_type = typename Time64Type::c_type; if (full_validation) { - c_type fullDay_us = 1000000 * 60 * 60 * 24; + c_type fullDay_us = 1000000ll * 60 * 60 * 24; c_type fullDay_ns = fullDay_us * 1000; return VisitArrayDataInline( data, From bd1209e638b642f977cab8148eb1c911159fda1c Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Wed, 22 Dec 2021 16:31:25 -0500 Subject: [PATCH 03/17] Resolving formatting issues and adding constexpr's --- cpp/src/arrow/array/validate.cc | 50 +++++++++++++++------------------ 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index 2c328ffe1bc..b99f0f1a305 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -167,17 +167,17 @@ struct ValidateArrayImpl { } Status Visit(const Date64Type& type) { - // check that data is divisible by 8.64e7 (= 1000ms * 60s * 60mins * 24hrs) RETURN_NOT_OK(ValidateFixedWidthBuffers()); - + using c_type = typename Date64Type::c_type; if (full_validation) { - c_type fullDay = 1000 * 60 * 60 * 24; + constexpr c_type full_day = 1000 * 60 * 60 * 24; return VisitArrayDataInline( data, [&](c_type date) { - if(date % fullDay != 0) { - return Status::Invalid(type, date, "ms does not represent a whole number of days"); + if (date % full_day != 0) { + return Status::Invalid(type, date, + "ms does not represent a whole number of days"); } return Status::OK(); }, @@ -187,27 +187,24 @@ struct ValidateArrayImpl { } Status Visit(const Time32Type& type) { - // check unit - // if unit is s => data must be within [0, 8.64e4) - // if unit is ms => data must be within [0, 8.64e7) RETURN_NOT_OK(ValidateFixedWidthBuffers()); - + using c_type = typename Time32Type::c_type; if (full_validation) { - c_type fullDay_s = 60 * 60 * 24; - c_type fullDay_ms = fullDay_s * 1000; + constexpr c_type full_day_s = 60 * 60 * 24; + constexpr c_type full_day_ms = full_day_s * 1000; return VisitArrayDataInline( data, [&](c_type time) { - if(type.unit() == TimeUnit::SECOND && (time < 0 || time >= fullDay_s)) { + if (type.unit() == TimeUnit::SECOND && (time < 0 || time >= full_day_s)) { return Status::Invalid(type, " ", time, - "s does not fit within the acceptable range of [0, ", - fullDay_s, ") s"); + "s does not fit within the acceptable range of ", + "[0, ", full_day_s, ") s"); } - if(type.unit() == TimeUnit::MILLI && (time < 0 || time >= fullDay_ms)) { + if (type.unit() == TimeUnit::MILLI && (time < 0 || time >= full_day_ms)) { return Status::Invalid(type, " ", time, - "ms does not fit within the acceptable range of [0, ", - fullDay_ms, ") ms"); + "ms does not fit within the acceptable range of ", + "[0, ", full_day_ms, ") ms"); } return Status::OK(); }, @@ -217,27 +214,24 @@ struct ValidateArrayImpl { } Status Visit(const Time64Type& type) { - // check unit - // if unit is us => data must be within [0, 8.64e10) - // if unit is ns => data must be within [0, 8.64e13) RETURN_NOT_OK(ValidateFixedWidthBuffers()); using c_type = typename Time64Type::c_type; if (full_validation) { - c_type fullDay_us = 1000000ll * 60 * 60 * 24; - c_type fullDay_ns = fullDay_us * 1000; + constexpr c_type full_day_us = 1000000LL * 60 * 60 * 24; + constexpr c_type full_day_ns = full_day_us * 1000; return VisitArrayDataInline( data, [&](c_type time) { - if(type.unit() == TimeUnit::MICRO && (time < 0 || time >= fullDay_us)) { + if (type.unit() == TimeUnit::MICRO && (time < 0 || time >= full_day_us)) { return Status::Invalid(type, " ", time, - "us does not fit within the acceptable range of [0, ", - fullDay_us, ") us"); + "us does not fit within the acceptable range of ", + "[0, ", full_day_us, ") us"); } - if(type.unit() == TimeUnit::NANO && (time < 0 || time >= fullDay_ns)) { + if (type.unit() == TimeUnit::NANO && (time < 0 || time >= full_day_ns)) { return Status::Invalid(type, " ", time, - "ns does not fit within the acceptable range of [0, ", - fullDay_ns, ") ns"); + "ns does not fit within the acceptable range of ", + "[0, ", full_day_ns, ") ns"); } return Status::OK(); }, From aeedda52a189ed50158eb800ddeb993370196862 Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Wed, 22 Dec 2021 16:46:25 -0500 Subject: [PATCH 04/17] Using kCamelCase for constants --- cpp/src/arrow/array/validate.cc | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index b99f0f1a305..0a4a8de3122 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -171,11 +171,11 @@ struct ValidateArrayImpl { using c_type = typename Date64Type::c_type; if (full_validation) { - constexpr c_type full_day = 1000 * 60 * 60 * 24; + constexpr c_type kFullDay = 1000 * 60 * 60 * 24; return VisitArrayDataInline( data, [&](c_type date) { - if (date % full_day != 0) { + if (date % kFullDay != 0) { return Status::Invalid(type, date, "ms does not represent a whole number of days"); } @@ -191,20 +191,20 @@ struct ValidateArrayImpl { using c_type = typename Time32Type::c_type; if (full_validation) { - constexpr c_type full_day_s = 60 * 60 * 24; - constexpr c_type full_day_ms = full_day_s * 1000; + constexpr c_type kFullDay_s = 60 * 60 * 24; + constexpr c_type kFullDay_ms = kFullDay_s * 1000; return VisitArrayDataInline( data, [&](c_type time) { - if (type.unit() == TimeUnit::SECOND && (time < 0 || time >= full_day_s)) { + if (type.unit() == TimeUnit::SECOND && (time < 0 || time >= kFullDay_s)) { return Status::Invalid(type, " ", time, "s does not fit within the acceptable range of ", - "[0, ", full_day_s, ") s"); + "[0, ", kFullDay_s, ") s"); } - if (type.unit() == TimeUnit::MILLI && (time < 0 || time >= full_day_ms)) { + if (type.unit() == TimeUnit::MILLI && (time < 0 || time >= kFullDay_ms)) { return Status::Invalid(type, " ", time, "ms does not fit within the acceptable range of ", - "[0, ", full_day_ms, ") ms"); + "[0, ", kFullDay_ms, ") ms"); } return Status::OK(); }, @@ -218,20 +218,20 @@ struct ValidateArrayImpl { using c_type = typename Time64Type::c_type; if (full_validation) { - constexpr c_type full_day_us = 1000000LL * 60 * 60 * 24; - constexpr c_type full_day_ns = full_day_us * 1000; + constexpr c_type kFullDay_us = 1000000LL * 60 * 60 * 24; + constexpr c_type kFullDay_ns = kFullDay_us * 1000; return VisitArrayDataInline( data, [&](c_type time) { - if (type.unit() == TimeUnit::MICRO && (time < 0 || time >= full_day_us)) { + if (type.unit() == TimeUnit::MICRO && (time < 0 || time >= kFullDay_us)) { return Status::Invalid(type, " ", time, "us does not fit within the acceptable range of ", - "[0, ", full_day_us, ") us"); + "[0, ", kFullDay_us, ") us"); } - if (type.unit() == TimeUnit::NANO && (time < 0 || time >= full_day_ns)) { + if (type.unit() == TimeUnit::NANO && (time < 0 || time >= kFullDay_ns)) { return Status::Invalid(type, " ", time, "ns does not fit within the acceptable range of ", - "[0, ", full_day_ns, ") ns"); + "[0, ", kFullDay_ns, ") ns"); } return Status::OK(); }, From 9400fbbe4968861fa1c0d4bdebbf54ed683c69ad Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Wed, 22 Dec 2021 16:51:48 -0500 Subject: [PATCH 05/17] Renaming time constants --- cpp/src/arrow/array/validate.cc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index 0a4a8de3122..747ac750ed3 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -191,20 +191,20 @@ struct ValidateArrayImpl { using c_type = typename Time32Type::c_type; if (full_validation) { - constexpr c_type kFullDay_s = 60 * 60 * 24; - constexpr c_type kFullDay_ms = kFullDay_s * 1000; + constexpr c_type kFullDaySeconds = 60 * 60 * 24; + constexpr c_type kFullDayMillis = kFullDaySeconds * 1000; return VisitArrayDataInline( data, [&](c_type time) { - if (type.unit() == TimeUnit::SECOND && (time < 0 || time >= kFullDay_s)) { + if (type.unit() == TimeUnit::SECOND && (time < 0 || time >= kFullDaySeconds)) { return Status::Invalid(type, " ", time, "s does not fit within the acceptable range of ", - "[0, ", kFullDay_s, ") s"); + "[0, ", kFullDaySeconds, ") s"); } - if (type.unit() == TimeUnit::MILLI && (time < 0 || time >= kFullDay_ms)) { + if (type.unit() == TimeUnit::MILLI && (time < 0 || time >= kFullDayMillis)) { return Status::Invalid(type, " ", time, "ms does not fit within the acceptable range of ", - "[0, ", kFullDay_ms, ") ms"); + "[0, ", kFullDayMillis, ") ms"); } return Status::OK(); }, @@ -218,20 +218,20 @@ struct ValidateArrayImpl { using c_type = typename Time64Type::c_type; if (full_validation) { - constexpr c_type kFullDay_us = 1000000LL * 60 * 60 * 24; - constexpr c_type kFullDay_ns = kFullDay_us * 1000; + constexpr c_type kFullDayMicro = 1000000LL * 60 * 60 * 24; + constexpr c_type kFullDayNano = kFullDayMicro * 1000; return VisitArrayDataInline( data, [&](c_type time) { - if (type.unit() == TimeUnit::MICRO && (time < 0 || time >= kFullDay_us)) { + if (type.unit() == TimeUnit::MICRO && (time < 0 || time >= kFullDayMicro)) { return Status::Invalid(type, " ", time, "us does not fit within the acceptable range of ", - "[0, ", kFullDay_us, ") us"); + "[0, ", kFullDayMicro, ") us"); } - if (type.unit() == TimeUnit::NANO && (time < 0 || time >= kFullDay_ns)) { + if (type.unit() == TimeUnit::NANO && (time < 0 || time >= kFullDayNano)) { return Status::Invalid(type, " ", time, "ns does not fit within the acceptable range of ", - "[0, ", kFullDay_ns, ") ns"); + "[0, ", kFullDayNano, ") ns"); } return Status::OK(); }, From f68c2a87a1a8fc349e2921f488759f6d6d43d60c Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Fri, 7 Jan 2022 01:11:37 -0500 Subject: [PATCH 06/17] Moving constexpr's and making small grammatical edits --- cpp/src/arrow/array/validate.cc | 39 +++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index 1fc47df151f..687814fec12 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -171,14 +171,14 @@ struct ValidateArrayImpl { Status Visit(const Date64Type& type) { RETURN_NOT_OK(ValidateFixedWidthBuffers()); - using c_type = typename Date64Type::c_type; if (full_validation) { - constexpr c_type kFullDay = 1000 * 60 * 60 * 24; + using c_type = typename Date64Type::c_type; return VisitArrayDataInline( data, [&](c_type date) { - if (date % kFullDay != 0) { - return Status::Invalid(type, date, + constexpr c_type kFullDayMillis = 1000 * 60 * 60 * 24; + if (date % kFullDayMillis != 0) { + return Status::Invalid(type, " ", date, "ms does not represent a whole number of days"); } return Status::OK(); @@ -191,22 +191,23 @@ struct ValidateArrayImpl { Status Visit(const Time32Type& type) { RETURN_NOT_OK(ValidateFixedWidthBuffers()); - using c_type = typename Time32Type::c_type; if (full_validation) { - constexpr c_type kFullDaySeconds = 60 * 60 * 24; - constexpr c_type kFullDayMillis = kFullDaySeconds * 1000; + using c_type = typename Time32Type::c_type; return VisitArrayDataInline( data, [&](c_type time) { - if (type.unit() == TimeUnit::SECOND && (time < 0 || time >= kFullDaySeconds)) { + constexpr c_type kFullDaySeconds = 60 * 60 * 24; + constexpr c_type kFullDayMillis = kFullDaySeconds * 1000; + if (type.unit() == TimeUnit::SECOND && + (time < 0 || time >= kFullDaySeconds)) { return Status::Invalid(type, " ", time, - "s does not fit within the acceptable range of ", - "[0, ", kFullDaySeconds, ") s"); + "s is not within the acceptable range of ", "[0, ", + kFullDaySeconds, ") s"); } if (type.unit() == TimeUnit::MILLI && (time < 0 || time >= kFullDayMillis)) { return Status::Invalid(type, " ", time, - "ms does not fit within the acceptable range of ", - "[0, ", kFullDayMillis, ") ms"); + "ms is not within the acceptable range of ", "[0, ", + kFullDayMillis, ") ms"); } return Status::OK(); }, @@ -218,22 +219,22 @@ struct ValidateArrayImpl { Status Visit(const Time64Type& type) { RETURN_NOT_OK(ValidateFixedWidthBuffers()); - using c_type = typename Time64Type::c_type; if (full_validation) { - constexpr c_type kFullDayMicro = 1000000LL * 60 * 60 * 24; - constexpr c_type kFullDayNano = kFullDayMicro * 1000; + using c_type = typename Time64Type::c_type; return VisitArrayDataInline( data, [&](c_type time) { + constexpr c_type kFullDayMicro = 1000000LL * 60 * 60 * 24; + constexpr c_type kFullDayNano = kFullDayMicro * 1000; if (type.unit() == TimeUnit::MICRO && (time < 0 || time >= kFullDayMicro)) { return Status::Invalid(type, " ", time, - "us does not fit within the acceptable range of ", - "[0, ", kFullDayMicro, ") us"); + "us is not within the acceptable range of ", "[0, ", + kFullDayMicro, ") us"); } if (type.unit() == TimeUnit::NANO && (time < 0 || time >= kFullDayNano)) { return Status::Invalid(type, " ", time, - "ns does not fit within the acceptable range of ", - "[0, ", kFullDayNano, ") ns"); + "ns is not within the acceptable range of ", "[0, ", + kFullDayNano, ") ns"); } return Status::OK(); }, From 827b6a42e7084c9c7f82eb4a2ff9395d8dd8a2cb Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Fri, 7 Jan 2022 01:50:14 -0500 Subject: [PATCH 07/17] More grammar corrections and updating array test constant for Date64Scalar --- cpp/src/arrow/array/array_test.cc | 2 +- cpp/src/arrow/array/validate.cc | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/array/array_test.cc b/cpp/src/arrow/array/array_test.cc index dc1bd624747..76b13a6dc73 100644 --- a/cpp/src/arrow/array/array_test.cc +++ b/cpp/src/arrow/array/array_test.cc @@ -534,7 +534,7 @@ static ScalarVector GetScalars() { std::make_shared(3), std::make_shared(3.0), std::make_shared(10), - std::make_shared(11), + std::make_shared(864000000), std::make_shared(1000, time32(TimeUnit::SECOND)), std::make_shared(1111, time64(TimeUnit::MICRO)), std::make_shared(1111, timestamp(TimeUnit::MILLI)), diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index 687814fec12..158d7a11858 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -179,7 +179,7 @@ struct ValidateArrayImpl { constexpr c_type kFullDayMillis = 1000 * 60 * 60 * 24; if (date % kFullDayMillis != 0) { return Status::Invalid(type, " ", date, - "ms does not represent a whole number of days"); + " does not represent a whole number of days"); } return Status::OK(); }, @@ -201,12 +201,12 @@ struct ValidateArrayImpl { if (type.unit() == TimeUnit::SECOND && (time < 0 || time >= kFullDaySeconds)) { return Status::Invalid(type, " ", time, - "s is not within the acceptable range of ", "[0, ", + " is not within the acceptable range of ", "[0, ", kFullDaySeconds, ") s"); } if (type.unit() == TimeUnit::MILLI && (time < 0 || time >= kFullDayMillis)) { return Status::Invalid(type, " ", time, - "ms is not within the acceptable range of ", "[0, ", + " is not within the acceptable range of ", "[0, ", kFullDayMillis, ") ms"); } return Status::OK(); @@ -228,12 +228,12 @@ struct ValidateArrayImpl { constexpr c_type kFullDayNano = kFullDayMicro * 1000; if (type.unit() == TimeUnit::MICRO && (time < 0 || time >= kFullDayMicro)) { return Status::Invalid(type, " ", time, - "us is not within the acceptable range of ", "[0, ", + " is not within the acceptable range of ", "[0, ", kFullDayMicro, ") us"); } if (type.unit() == TimeUnit::NANO && (time < 0 || time >= kFullDayNano)) { return Status::Invalid(type, " ", time, - "ns is not within the acceptable range of ", "[0, ", + " is not within the acceptable range of ", "[0, ", kFullDayNano, ") ns"); } return Status::OK(); From 4ace4965634ac519302c7aaef752a2ee5986c700 Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Sat, 8 Jan 2022 00:56:34 -0500 Subject: [PATCH 08/17] Added random generation handling for temporal types --- cpp/src/arrow/testing/random.cc | 59 ++++++++++++++++++++++++++++++--- cpp/src/arrow/testing/random.h | 14 ++++++++ 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/testing/random.cc b/cpp/src/arrow/testing/random.cc index c5ffe9c2aa8..0d2be9532a9 100644 --- a/cpp/src/arrow/testing/random.cc +++ b/cpp/src/arrow/testing/random.cc @@ -102,6 +102,12 @@ struct GenerateOptions { std::generate(data, data + n, [&] { return static_cast(dist(rng)); }); } + void GenerateFullDayNoNan(uint8_t* buffer, size_t n) { + int64_t* data = reinterpret_cast(buffer); + constexpr int64_t kFullDayMillis = 1000 * 60 * 60 * 24; + std::for_each(data, data + n, [&] (int64_t v) { return v * kFullDayMillis; }); + } + void GenerateBitmap(uint8_t* buffer, size_t n, int64_t* null_count) { int64_t count = 0; pcg32_fast rng(seed_++); @@ -182,6 +188,9 @@ static std::shared_ptr> GenerateNumericArray(int64_t siz buffers[1] = *AllocateBuffer(sizeof(CType) * size); options.GenerateData(buffers[1]->mutable_data(), size); + if(std::is_same::value) { + options.GenerateFullDayNoNan(buffers[1]->mutable_data(), size); + } auto array_data = ArrayData::Make(type, size, buffers, null_count); return std::make_shared>(array_data); @@ -192,7 +201,7 @@ static std::shared_ptr> GenerateNumericArray(int64_t siz double probability) { \ using OptionType = GenerateOptions; \ OptionType options(seed(), min, max, probability); \ - return GenerateNumericArray(size, options); \ + return GenerateNumericArray(size, options); \ } #define PRIMITIVE_RAND_INTEGER_IMPL(Name, CType, ArrowType) \ @@ -211,6 +220,14 @@ PRIMITIVE_RAND_INTEGER_IMPL(Int64, int64_t, Int64Type) // Generate 16bit values for half-float PRIMITIVE_RAND_INTEGER_IMPL(Float16, int16_t, HalfFloatType) +std::shared_ptr RandomArrayGenerator::Date64(int64_t size, int64_t min, + int64_t max, double null_probability) { + using OptionType = + GenerateOptions>; + OptionType options(seed(), min, max, null_probability); + return GenerateNumericArray(size, options); +} + std::shared_ptr RandomArrayGenerator::Float32(int64_t size, float min, float max, double null_probability, double nan_probability) { @@ -772,12 +789,46 @@ std::shared_ptr RandomArrayGenerator::ArrayOf(const Field& field, int64_t } GENERATE_INTEGRAL_CASE_VIEW(Int32Type, Date32Type); - GENERATE_INTEGRAL_CASE_VIEW(Int64Type, Date64Type); GENERATE_INTEGRAL_CASE_VIEW(Int64Type, TimestampType); - GENERATE_INTEGRAL_CASE_VIEW(Int32Type, Time32Type); - GENERATE_INTEGRAL_CASE_VIEW(Int64Type, Time64Type); GENERATE_INTEGRAL_CASE_VIEW(Int32Type, MonthIntervalType); + case Type::type::DATE64: { + std::shared_ptr type = + std::dynamic_pointer_cast(field.type()); + using c_type = typename Date64Type::c_type; + constexpr c_type kFullDayMillis = 1000 * 60 * 60 * 24; + constexpr c_type min_value = std::numeric_limits::min() / kFullDayMillis; + constexpr c_type max_value = std::numeric_limits::max() / kFullDayMillis; + + return *Numeric(length, min_value, max_value, null_probability) + ->View(field.type()); + } + + case Type::type::TIME32: { + std::shared_ptr type = + std::dynamic_pointer_cast(field.type()); + using c_type = typename Time32Type::c_type; + const c_type min_value = 0; + const c_type max_value = (type->unit() == TimeUnit::SECOND) + ? (60 * 60 * 24 - 1) + : (1000 * 60 * 60 * 24 - 1); + + return *Numeric(length, min_value, max_value, null_probability) + ->View(field.type()); + } + + case Type::type::TIME64: { + std::shared_ptr type = + std::dynamic_pointer_cast(field.type()); + using c_type = typename Time64Type::c_type; + const c_type min_value = 0; + const c_type max_value = (type->unit() == TimeUnit::MICRO) + ? (1000000LL * 60 * 60 * 24 - 1) + : (1000000000LL * 60 * 60 * 24 - 1); + return *Numeric(length, min_value, max_value, null_probability) + ->View(field.type()); + } + // This isn't as flexible as it could be, but the array-of-structs layout of this // type means it's not a (useful) composition of other generators GENERATE_INTEGRAL_CASE_VIEW(Int64Type, DayTimeIntervalType); diff --git a/cpp/src/arrow/testing/random.h b/cpp/src/arrow/testing/random.h index 5e486d90ac3..07345babb29 100644 --- a/cpp/src/arrow/testing/random.h +++ b/cpp/src/arrow/testing/random.h @@ -183,6 +183,17 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator { /// \return a generated Array std::shared_ptr Float64(int64_t size, double min, double max, double null_probability = 0, double nan_probability = 0); + + /// \brief Generate a random Date64Array + /// + /// \param[in] size the size of the array to generate + /// \param[in] min the lower bound of the uniform distribution + /// \param[in] max the upper bound of the uniform distribution + /// \param[in] null_probability the probability of a value being null + /// + /// \return a generated Array + std::shared_ptr Date64(int64_t size, int64_t min, int64_t max, + double null_probability = 0); template std::shared_ptr Numeric(int64_t size, CType min, CType max, @@ -221,6 +232,9 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator { case Type::DOUBLE: return Float64(size, static_cast(min), static_cast(max), null_probability); + case Type::DATE64: + return Date64(size, static_cast(min), static_cast(max), + null_probability); default: return nullptr; } From 36c3dfcb92b4bff06b604474efe9cfc0b758e0ff Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Sat, 8 Jan 2022 01:17:25 -0500 Subject: [PATCH 09/17] Formatting code properly --- cpp/src/arrow/testing/random.cc | 12 ++++++------ cpp/src/arrow/testing/random.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cpp/src/arrow/testing/random.cc b/cpp/src/arrow/testing/random.cc index 0d2be9532a9..bf9658f24bd 100644 --- a/cpp/src/arrow/testing/random.cc +++ b/cpp/src/arrow/testing/random.cc @@ -105,7 +105,7 @@ struct GenerateOptions { void GenerateFullDayNoNan(uint8_t* buffer, size_t n) { int64_t* data = reinterpret_cast(buffer); constexpr int64_t kFullDayMillis = 1000 * 60 * 60 * 24; - std::for_each(data, data + n, [&] (int64_t v) { return v * kFullDayMillis; }); + std::for_each(data, data + n, [&](int64_t v) { return v * kFullDayMillis; }); } void GenerateBitmap(uint8_t* buffer, size_t n, int64_t* null_count) { @@ -188,7 +188,7 @@ static std::shared_ptr> GenerateNumericArray(int64_t siz buffers[1] = *AllocateBuffer(sizeof(CType) * size); options.GenerateData(buffers[1]->mutable_data(), size); - if(std::is_same::value) { + if (std::is_same::value) { options.GenerateFullDayNoNan(buffers[1]->mutable_data(), size); } @@ -201,7 +201,7 @@ static std::shared_ptr> GenerateNumericArray(int64_t siz double probability) { \ using OptionType = GenerateOptions; \ OptionType options(seed(), min, max, probability); \ - return GenerateNumericArray(size, options); \ + return GenerateNumericArray(size, options); \ } #define PRIMITIVE_RAND_INTEGER_IMPL(Name, CType, ArrowType) \ @@ -221,9 +221,9 @@ PRIMITIVE_RAND_INTEGER_IMPL(Int64, int64_t, Int64Type) PRIMITIVE_RAND_INTEGER_IMPL(Float16, int16_t, HalfFloatType) std::shared_ptr RandomArrayGenerator::Date64(int64_t size, int64_t min, - int64_t max, double null_probability) { - using OptionType = - GenerateOptions>; + int64_t max, + double null_probability) { + using OptionType = GenerateOptions>; OptionType options(seed(), min, max, null_probability); return GenerateNumericArray(size, options); } diff --git a/cpp/src/arrow/testing/random.h b/cpp/src/arrow/testing/random.h index 07345babb29..00d50f9bc08 100644 --- a/cpp/src/arrow/testing/random.h +++ b/cpp/src/arrow/testing/random.h @@ -183,7 +183,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator { /// \return a generated Array std::shared_ptr Float64(int64_t size, double min, double max, double null_probability = 0, double nan_probability = 0); - + /// \brief Generate a random Date64Array /// /// \param[in] size the size of the array to generate @@ -193,7 +193,7 @@ class ARROW_TESTING_EXPORT RandomArrayGenerator { /// /// \return a generated Array std::shared_ptr Date64(int64_t size, int64_t min, int64_t max, - double null_probability = 0); + double null_probability = 0); template std::shared_ptr Numeric(int64_t size, CType min, CType max, From 6d7189be0aa0426c3c24d640ab7c05b22a225f1c Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Sat, 8 Jan 2022 17:54:26 -0500 Subject: [PATCH 10/17] Changing test constants to reflect new restrictions --- cpp/src/arrow/c/bridge_test.cc | 4 ++-- cpp/src/arrow/ipc/json_simple_test.cc | 8 ++++---- cpp/src/arrow/ipc/test_common.cc | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cpp/src/arrow/c/bridge_test.cc b/cpp/src/arrow/c/bridge_test.cc index 2c20c54cfe2..39e0eab68d4 100644 --- a/cpp/src/arrow/c/bridge_test.cc +++ b/cpp/src/arrow/c/bridge_test.cc @@ -1988,7 +1988,7 @@ TEST_F(TestArrayImport, Temporal) { FillPrimitive(3, 0, 0, primitive_buffers_no_nulls7); CheckImport(ArrayFromJSON(date32(), "[1234, 5678, 9012]")); FillPrimitive(3, 0, 0, primitive_buffers_no_nulls8); - CheckImport(ArrayFromJSON(date64(), "[123456789, 987654321, -123456789]")); + CheckImport(ArrayFromJSON(date64(), "[86400000, 172800000, -86400000]")); FillPrimitive(2, 0, 0, primitive_buffers_no_nulls7); CheckImport(ArrayFromJSON(time32(TimeUnit::SECOND), "[1234, 5678]")); @@ -2027,7 +2027,7 @@ TEST_F(TestArrayImport, Temporal) { FillPrimitive(3, -1, 0, primitive_buffers_nulls7); CheckImport(ArrayFromJSON(date32(), "[1234, null, 9012]")); FillPrimitive(3, -1, 0, primitive_buffers_nulls8); - CheckImport(ArrayFromJSON(date64(), "[123456789, null, -123456789]")); + CheckImport(ArrayFromJSON(date64(), "[86400000, null, -86400000]")); FillPrimitive(2, -1, 0, primitive_buffers_nulls8); CheckImport(ArrayFromJSON(time64(TimeUnit::NANO), "[123456789, null]")); FillPrimitive(2, -1, 0, primitive_buffers_nulls8); diff --git a/cpp/src/arrow/ipc/json_simple_test.cc b/cpp/src/arrow/ipc/json_simple_test.cc index c6f66d003f4..fa9bc742100 100644 --- a/cpp/src/arrow/ipc/json_simple_test.cc +++ b/cpp/src/arrow/ipc/json_simple_test.cc @@ -428,8 +428,8 @@ TEST(TestDate, Basics) { auto type = date32(); AssertJSONArray(type, R"([5, null, 42])", {true, false, true}, {5, 0, 42}); type = date64(); - AssertJSONArray(type, R"([1, null, 9999999999999])", {true, false, true}, - {1, 0, 9999999999999LL}); + AssertJSONArray(type, R"([86400000, null, 172800000])", {true, false, true}, + {86400000, 0, 172800000}); } TEST(TestTime, Basics) { @@ -439,8 +439,8 @@ TEST(TestTime, Basics) { AssertJSONArray(type, R"([5, null, 42])", {true, false, true}, {5, 0, 42}); type = time64(TimeUnit::MICRO); - AssertJSONArray(type, R"([1, null, 9999999999999])", {true, false, true}, - {1, 0, 9999999999999LL}); + AssertJSONArray(type, R"([1, null, 9999999999])", {true, false, true}, + {1, 0, 9999999999LL}); type = time64(TimeUnit::NANO); AssertJSONArray(type, R"([1, null, 9999999999999])", {true, false, true}, {1, 0, 9999999999999LL}); diff --git a/cpp/src/arrow/ipc/test_common.cc b/cpp/src/arrow/ipc/test_common.cc index d7c1d852b88..29e3b60c3c4 100644 --- a/cpp/src/arrow/ipc/test_common.cc +++ b/cpp/src/arrow/ipc/test_common.cc @@ -811,9 +811,9 @@ Status MakeDates(std::shared_ptr* out) { std::shared_ptr date32_array; ArrayFromVector(is_valid, date32_values, &date32_array); - std::vector date64_values = {1489269000000, 1489270000000, 1489271000000, - 1489272000000, 1489272000000, 1489273000000, - 1489274000000}; + std::vector date64_values = {86400000, 172800000, 259200000, + 1489272000000, 345600000, 432000000, + 518400000}; std::shared_ptr date64_array; ArrayFromVector(is_valid, date64_values, &date64_array); @@ -877,8 +877,8 @@ Status MakeTimes(std::shared_ptr* out) { auto f3 = field("f3", time64(TimeUnit::NANO)); auto schema = ::arrow::schema({f0, f1, f2, f3}); - std::vector t32_values = {1489269000, 1489270000, 1489271000, - 1489272000, 1489272000, 1489273000}; + std::vector t32_values = {14896, 14897, 14892, + 1489272000, 14893, 14895}; std::vector t64_values = {1489269000000, 1489270000000, 1489271000000, 1489272000000, 1489272000000, 1489273000000}; From d8be683bc43ce867e6c96072848fdfb8a993676c Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Sun, 9 Jan 2022 21:45:02 -0500 Subject: [PATCH 11/17] Added new test buffers for date64 --- cpp/src/arrow/c/bridge_test.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/c/bridge_test.cc b/cpp/src/arrow/c/bridge_test.cc index 39e0eab68d4..bb722c52b67 100644 --- a/cpp/src/arrow/c/bridge_test.cc +++ b/cpp/src/arrow/c/bridge_test.cc @@ -1668,6 +1668,7 @@ static const float data_buffer5[] = {0.0f, 1.5f, -2.0f, 3.0f, 4.0f, 5.0f}; static const double data_buffer6[] = {0.0, 1.5, -2.0, 3.0, 4.0, 5.0}; static const int32_t data_buffer7[] = {1234, 5678, 9012, 3456}; static const int64_t data_buffer8[] = {123456789, 987654321, -123456789, -987654321}; +static const int64_t date64_data_buffer8[] = {86400000, 172800000, -86400000, -172800000}; #if ARROW_LITTLE_ENDIAN static const void* primitive_buffers_no_nulls1_8[2] = {nullptr, data_buffer1}; static const void* primitive_buffers_no_nulls1_16[2] = {nullptr, data_buffer1}; @@ -1699,6 +1700,9 @@ static const void* primitive_buffers_nulls7[2] = {bits_buffer1, data_buffer7}; static const void* primitive_buffers_no_nulls8[2] = {nullptr, data_buffer8}; static const void* primitive_buffers_nulls8[2] = {bits_buffer1, data_buffer8}; +static const void* date64_buffers_no_nulls8[2] = {nullptr, date64_data_buffer8}; +static const void* date64_buffers_nulls8[2] = {bits_buffer1, date64_data_buffer8}; + static const int64_t timestamp_data_buffer1[] = {0, 951782400, -2203977600LL}; static const int64_t timestamp_data_buffer2[] = {0, 951782400000LL, -2203977600000LL}; static const int64_t timestamp_data_buffer3[] = {0, 951782400000000LL, @@ -1987,7 +1991,7 @@ TEST_F(TestArrayImport, Primitive) { TEST_F(TestArrayImport, Temporal) { FillPrimitive(3, 0, 0, primitive_buffers_no_nulls7); CheckImport(ArrayFromJSON(date32(), "[1234, 5678, 9012]")); - FillPrimitive(3, 0, 0, primitive_buffers_no_nulls8); + FillPrimitive(3, 0, 0, date64_buffers_no_nulls8); CheckImport(ArrayFromJSON(date64(), "[86400000, 172800000, -86400000]")); FillPrimitive(2, 0, 0, primitive_buffers_no_nulls7); @@ -2026,7 +2030,7 @@ TEST_F(TestArrayImport, Temporal) { // With nulls FillPrimitive(3, -1, 0, primitive_buffers_nulls7); CheckImport(ArrayFromJSON(date32(), "[1234, null, 9012]")); - FillPrimitive(3, -1, 0, primitive_buffers_nulls8); + FillPrimitive(3, -1, 0, date64_buffers_nulls8); CheckImport(ArrayFromJSON(date64(), "[86400000, null, -86400000]")); FillPrimitive(2, -1, 0, primitive_buffers_nulls8); CheckImport(ArrayFromJSON(time64(TimeUnit::NANO), "[123456789, null]")); From 997c7a14e0a0a08e065d33caa1c6c7e553228010 Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Mon, 10 Jan 2022 20:58:55 -0500 Subject: [PATCH 12/17] Fixed random number generation and constants for testing date64 --- cpp/src/arrow/ipc/test_common.cc | 8 +++----- cpp/src/arrow/testing/random.cc | 30 +++++++++++++----------------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/cpp/src/arrow/ipc/test_common.cc b/cpp/src/arrow/ipc/test_common.cc index 29e3b60c3c4..3130bc81049 100644 --- a/cpp/src/arrow/ipc/test_common.cc +++ b/cpp/src/arrow/ipc/test_common.cc @@ -811,9 +811,8 @@ Status MakeDates(std::shared_ptr* out) { std::shared_ptr date32_array; ArrayFromVector(is_valid, date32_values, &date32_array); - std::vector date64_values = {86400000, 172800000, 259200000, - 1489272000000, 345600000, 432000000, - 518400000}; + std::vector date64_values = {86400000, 172800000, 259200000, 1489272000000, + 345600000, 432000000, 518400000}; std::shared_ptr date64_array; ArrayFromVector(is_valid, date64_values, &date64_array); @@ -877,8 +876,7 @@ Status MakeTimes(std::shared_ptr* out) { auto f3 = field("f3", time64(TimeUnit::NANO)); auto schema = ::arrow::schema({f0, f1, f2, f3}); - std::vector t32_values = {14896, 14897, 14892, - 1489272000, 14893, 14895}; + std::vector t32_values = {14896, 14897, 14892, 1489272000, 14893, 14895}; std::vector t64_values = {1489269000000, 1489270000000, 1489271000000, 1489272000000, 1489272000000, 1489273000000}; diff --git a/cpp/src/arrow/testing/random.cc b/cpp/src/arrow/testing/random.cc index bf9658f24bd..fb60731babb 100644 --- a/cpp/src/arrow/testing/random.cc +++ b/cpp/src/arrow/testing/random.cc @@ -102,12 +102,6 @@ struct GenerateOptions { std::generate(data, data + n, [&] { return static_cast(dist(rng)); }); } - void GenerateFullDayNoNan(uint8_t* buffer, size_t n) { - int64_t* data = reinterpret_cast(buffer); - constexpr int64_t kFullDayMillis = 1000 * 60 * 60 * 24; - std::for_each(data, data + n, [&](int64_t v) { return v * kFullDayMillis; }); - } - void GenerateBitmap(uint8_t* buffer, size_t n, int64_t* null_count) { int64_t count = 0; pcg32_fast rng(seed_++); @@ -175,6 +169,12 @@ std::shared_ptr RandomArrayGenerator::Boolean(int64_t size, return std::make_shared(array_data); } +void GenerateFullDayMillisNoNan(uint8_t* buffer, size_t n) { + int64_t* data = reinterpret_cast(buffer); + constexpr int64_t kFullDayMillis = 1000 * 60 * 60 * 24; + std::for_each(data, data + n, [&](int64_t& v) { return v *= kFullDayMillis; }); +} + template static std::shared_ptr> GenerateNumericArray(int64_t size, OptionType options) { @@ -189,7 +189,7 @@ static std::shared_ptr> GenerateNumericArray(int64_t siz buffers[1] = *AllocateBuffer(sizeof(CType) * size); options.GenerateData(buffers[1]->mutable_data(), size); if (std::is_same::value) { - options.GenerateFullDayNoNan(buffers[1]->mutable_data(), size); + GenerateFullDayMillisNoNan(buffers[1]->mutable_data(), size); } auto array_data = ArrayData::Make(type, size, buffers, null_count); @@ -793,8 +793,6 @@ std::shared_ptr RandomArrayGenerator::ArrayOf(const Field& field, int64_t GENERATE_INTEGRAL_CASE_VIEW(Int32Type, MonthIntervalType); case Type::type::DATE64: { - std::shared_ptr type = - std::dynamic_pointer_cast(field.type()); using c_type = typename Date64Type::c_type; constexpr c_type kFullDayMillis = 1000 * 60 * 60 * 24; constexpr c_type min_value = std::numeric_limits::min() / kFullDayMillis; @@ -805,26 +803,24 @@ std::shared_ptr RandomArrayGenerator::ArrayOf(const Field& field, int64_t } case Type::type::TIME32: { - std::shared_ptr type = - std::dynamic_pointer_cast(field.type()); + TimeUnit::type unit = std::dynamic_pointer_cast(field.type())->unit(); using c_type = typename Time32Type::c_type; const c_type min_value = 0; - const c_type max_value = (type->unit() == TimeUnit::SECOND) - ? (60 * 60 * 24 - 1) - : (1000 * 60 * 60 * 24 - 1); + const c_type max_value = + (unit == TimeUnit::SECOND) ? (60 * 60 * 24 - 1) : (1000 * 60 * 60 * 24 - 1); return *Numeric(length, min_value, max_value, null_probability) ->View(field.type()); } case Type::type::TIME64: { - std::shared_ptr type = - std::dynamic_pointer_cast(field.type()); + TimeUnit::type unit = std::dynamic_pointer_cast(field.type())->unit(); using c_type = typename Time64Type::c_type; const c_type min_value = 0; - const c_type max_value = (type->unit() == TimeUnit::MICRO) + const c_type max_value = (unit == TimeUnit::MICRO) ? (1000000LL * 60 * 60 * 24 - 1) : (1000000000LL * 60 * 60 * 24 - 1); + return *Numeric(length, min_value, max_value, null_probability) ->View(field.type()); } From e409c73a7d179d296595942f6201fad68a372654 Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Thu, 27 Jan 2022 20:47:14 -0500 Subject: [PATCH 13/17] Changes to the last unit tests for date64 restrictions --- .../compute/kernels/hash_aggregate_test.cc | 63 +- .../arrow/compute/kernels/scalar_cast_test.cc | 5 +- .../compute/kernels/scalar_compare_test.cc | 446 +++++--- .../compute/kernels/scalar_if_else_test.cc | 455 +++++--- .../arrow/compute/kernels/vector_hash_test.cc | 131 ++- .../compute/kernels/vector_replace_test.cc | 971 +++++++++++++----- 6 files changed, 1492 insertions(+), 579 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/hash_aggregate_test.cc b/cpp/src/arrow/compute/kernels/hash_aggregate_test.cc index 0cc24b73b7b..53e42ab1568 100644 --- a/cpp/src/arrow/compute/kernels/hash_aggregate_test.cc +++ b/cpp/src/arrow/compute/kernels/hash_aggregate_test.cc @@ -1616,14 +1616,12 @@ TEST(GroupBy, MinMaxTypes) { types.insert(types.end(), NumericTypes().begin(), NumericTypes().end()); types.insert(types.end(), TemporalTypes().begin(), TemporalTypes().end()); types.push_back(month_interval()); - for (const auto& ty : types) { - SCOPED_TRACE(ty->ToString()); - auto in_schema = schema({field("argument0", ty), field("key", int64())}); - auto table = TableFromJSON(in_schema, {R"([ + + const std::vector default_table = {R"([ [1, 1], [null, 1] ])", - R"([ + R"([ [0, 2], [null, 3], [3, 4], @@ -1632,11 +1630,54 @@ TEST(GroupBy, MinMaxTypes) { [3, 1], [0, 2] ])", - R"([ + R"([ [0, 2], [1, null], [null, 3] -])"}); +])"}; + + const std::vector date64_table = {R"([ + [86400000, 1], + [null, 1] +])", + R"([ + [0, 2], + [null, 3], + [259200000, 4], + [432000000, 4], + [345600000, null], + [259200000, 1], + [0, 2] +])", + R"([ + [0, 2], + [86400000, null], + [null, 3] +])"}; + + const std::string default_expected = + R"([ + [{"min": 1, "max": 3}, 1], + [{"min": 0, "max": 0}, 2], + [{"min": null, "max": null}, 3], + [{"min": 3, "max": 5}, 4], + [{"min": 1, "max": 4}, null] + ])"; + + const std::string date64_expected = + R"([ + [{"min": 86400000, "max": 259200000}, 1], + [{"min": 0, "max": 0}, 2], + [{"min": null, "max": null}, 3], + [{"min": 259200000, "max": 432000000}, 4], + [{"min": 86400000, "max": 345600000}, null] + ])"; + + for (const auto& ty : types) { + SCOPED_TRACE(ty->ToString()); + auto in_schema = schema({field("argument0", ty), field("key", int64())}); + auto table = + TableFromJSON(in_schema, (ty->name() == "date64") ? date64_table : default_table); ASSERT_OK_AND_ASSIGN( Datum aggregated_and_grouped, @@ -1652,13 +1693,7 @@ TEST(GroupBy, MinMaxTypes) { field("hash_min_max", struct_({field("min", ty), field("max", ty)})), field("key_0", int64()), }), - R"([ - [{"min": 1, "max": 3}, 1], - [{"min": 0, "max": 0}, 2], - [{"min": null, "max": null}, 3], - [{"min": 3, "max": 5}, 4], - [{"min": 1, "max": 4}, null] - ])"), + (ty->name() == "date64") ? date64_expected : default_expected), aggregated_and_grouped, /*verbose=*/true); } diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc index 226452404d0..cb2eb62f6f2 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc @@ -1670,10 +1670,11 @@ TEST(Cast, DateZeroCopy) { date64(), int64(), // ARROW-1773: cast to int64 }) { - CheckCastZeroCopy(ArrayFromJSON(date64(), "[0, null, 2000, 1000, 0]"), + CheckCastZeroCopy(ArrayFromJSON(date64(), "[0, null, 172800000, 86400000, 0]"), zero_copy_to_type); } - CheckCastZeroCopy(ArrayFromJSON(int64(), "[0, null, 2000, 1000, 0]"), date64()); + CheckCastZeroCopy(ArrayFromJSON(int64(), "[0, null, 172800000, 86400000, 0]"), + date64()); } TEST(Cast, DurationToDuration) { diff --git a/cpp/src/arrow/compute/kernels/scalar_compare_test.cc b/cpp/src/arrow/compute/kernels/scalar_compare_test.cc index b6749664010..f0f2d7e3679 100644 --- a/cpp/src/arrow/compute/kernels/scalar_compare_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_compare_test.cc @@ -1229,70 +1229,166 @@ TYPED_TEST(TestVarArgsCompareNumeric, MinElementWise) { this->AssertNullScalar(MinElementWise, {}); this->AssertNullScalar(MinElementWise, {this->scalar("null"), this->scalar("null")}); - this->Assert(MinElementWise, this->scalar("0"), {this->scalar("0")}); - this->Assert(MinElementWise, this->scalar("0"), - {this->scalar("2"), this->scalar("0"), this->scalar("1")}); - this->Assert( - MinElementWise, this->scalar("0"), - {this->scalar("2"), this->scalar("0"), this->scalar("1"), this->scalar("null")}); - this->Assert(MinElementWise, this->scalar("1"), - {this->scalar("null"), this->scalar("null"), this->scalar("1"), - this->scalar("null")}); - - this->Assert(MinElementWise, (this->array("[]")), {this->array("[]")}); - this->Assert(MinElementWise, this->array("[1, 2, 3, null]"), - {this->array("[1, 2, 3, null]")}); - - this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), - {this->array("[1, 2, 3, 4]"), this->scalar("2")}); - this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), - {this->array("[1, null, 3, 4]"), this->scalar("2")}); - this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), - {this->array("[1, null, 3, 4]"), this->scalar("2"), this->scalar("4")}); - this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), - {this->array("[1, null, 3, 4]"), this->scalar("null"), this->scalar("2")}); - - this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), - {this->array("[1, 2, 3, 4]"), this->array("[2, 2, 2, 2]")}); - this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), - {this->array("[1, 2, 3, 4]"), this->array("[2, null, 2, 2]")}); - this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), - {this->array("[1, null, 3, 4]"), this->array("[2, 2, 2, 2]")}); - - this->Assert(MinElementWise, this->array("[1, 2, null, 6]"), - {this->array("[1, 2, null, null]"), this->array("[4, null, null, 6]")}); - this->Assert(MinElementWise, this->array("[1, 2, null, 6]"), - {this->array("[4, null, null, 6]"), this->array("[1, 2, null, null]")}); - this->Assert(MinElementWise, this->array("[1, 2, 3, 4]"), - {this->array("[1, 2, 3, 4]"), this->array("[null, null, null, null]")}); - this->Assert(MinElementWise, this->array("[1, 2, 3, 4]"), - {this->array("[null, null, null, null]"), this->array("[1, 2, 3, 4]")}); - - this->Assert(MinElementWise, this->array("[1, 1, 1, 1]"), - {this->scalar("1"), this->array("[1, 2, 3, 4]")}); - this->Assert(MinElementWise, this->array("[1, 1, 1, 1]"), - {this->scalar("1"), this->array("[null, null, null, null]")}); - this->Assert(MinElementWise, this->array("[1, 1, 1, 1]"), - {this->scalar("null"), this->array("[1, 1, 1, 1]")}); - this->Assert(MinElementWise, this->array("[null, null, null, null]"), - {this->scalar("null"), this->array("[null, null, null, null]")}); - - // Test null handling - this->element_wise_aggregate_options_.skip_nulls = false; - this->AssertNullScalar(MinElementWise, {this->scalar("null"), this->scalar("null")}); - this->AssertNullScalar(MinElementWise, {this->scalar("0"), this->scalar("null")}); - - this->Assert(MinElementWise, this->array("[1, null, 2, 2]"), - {this->array("[1, null, 3, 4]"), this->scalar("2"), this->scalar("4")}); - this->Assert(MinElementWise, this->array("[null, null, null, null]"), - {this->array("[1, null, 3, 4]"), this->scalar("null"), this->scalar("2")}); - this->Assert(MinElementWise, this->array("[1, null, 2, 2]"), - {this->array("[1, 2, 3, 4]"), this->array("[2, null, 2, 2]")}); - - this->Assert(MinElementWise, this->array("[null, null, null, null]"), - {this->scalar("1"), this->array("[null, null, null, null]")}); - this->Assert(MinElementWise, this->array("[null, null, null, null]"), - {this->scalar("null"), this->array("[1, 1, 1, 1]")}); + if (std::is_same::value) { + this->Assert(MinElementWise, this->scalar("0"), {this->scalar("0")}); + this->Assert( + MinElementWise, this->scalar("0"), + {this->scalar("172800000"), this->scalar("0"), this->scalar("86400000")}); + this->Assert(MinElementWise, this->scalar("0"), + {this->scalar("172800000"), this->scalar("0"), this->scalar("86400000"), + this->scalar("null")}); + this->Assert(MinElementWise, this->scalar("86400000"), + {this->scalar("null"), this->scalar("null"), this->scalar("86400000"), + this->scalar("null")}); + + this->Assert(MinElementWise, (this->array("[]")), {this->array("[]")}); + this->Assert(MinElementWise, this->array("[86400000, 172800000, 259200000, null]"), + {this->array("[86400000, 172800000, 259200000, null]")}); + + this->Assert(MinElementWise, + this->array("[86400000, 172800000, 172800000, 172800000]"), + {this->array("[86400000, 172800000, 259200000, 345600000]"), + this->scalar("172800000")}); + this->Assert(MinElementWise, + this->array("[86400000, 172800000, 172800000, 172800000]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->scalar("172800000")}); + this->Assert(MinElementWise, + this->array("[86400000, 172800000, 172800000, 172800000]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->scalar("172800000"), this->scalar("345600000")}); + this->Assert(MinElementWise, + this->array("[86400000, 172800000, 172800000, 172800000]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->scalar("null"), this->scalar("172800000")}); + + this->Assert(MinElementWise, + this->array("[86400000, 172800000, 172800000, 172800000]"), + {this->array("[86400000, 172800000, 259200000, 345600000]"), + this->array("[172800000, 172800000, 172800000, 172800000]")}); + this->Assert(MinElementWise, + this->array("[86400000, 172800000, 172800000, 172800000]"), + {this->array("[86400000, 172800000, 259200000, 345600000]"), + this->array("[172800000, null, 172800000, 172800000]")}); + this->Assert(MinElementWise, + this->array("[86400000, 172800000, 172800000, 172800000]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->array("[172800000, 172800000, 172800000, 172800000]")}); + + this->Assert(MinElementWise, this->array("[86400000, 172800000, null, 518400000]"), + {this->array("[86400000, 172800000, null, null]"), + this->array("[345600000, null, null, 518400000]")}); + this->Assert(MinElementWise, this->array("[86400000, 172800000, null, 518400000]"), + {this->array("[345600000, null, null, 518400000]"), + this->array("[86400000, 172800000, null, null]")}); + this->Assert(MinElementWise, + this->array("[86400000, 172800000, 259200000, 345600000]"), + {this->array("[86400000, 172800000, 259200000, 345600000]"), + this->array("[null, null, null, null]")}); + this->Assert(MinElementWise, + this->array("[86400000, 172800000, 259200000, 345600000]"), + {this->array("[null, null, null, null]"), + this->array("[86400000, 172800000, 259200000, 345600000]")}); + + this->Assert(MinElementWise, this->array("[86400000, 86400000, 86400000, 86400000]"), + {this->scalar("86400000"), + this->array("[86400000, 172800000, 259200000, 345600000]")}); + this->Assert(MinElementWise, this->array("[86400000, 86400000, 86400000, 86400000]"), + {this->scalar("86400000"), this->array("[null, null, null, null]")}); + this->Assert( + MinElementWise, this->array("[86400000, 86400000, 86400000, 86400000]"), + {this->scalar("null"), this->array("[86400000, 86400000, 86400000, 86400000]")}); + this->Assert(MinElementWise, this->array("[null, null, null, null]"), + {this->scalar("null"), this->array("[null, null, null, null]")}); + + // Test null handling + this->element_wise_aggregate_options_.skip_nulls = false; + this->AssertNullScalar(MinElementWise, {this->scalar("null"), this->scalar("null")}); + this->AssertNullScalar(MinElementWise, {this->scalar("0"), this->scalar("null")}); + + this->Assert(MinElementWise, this->array("[86400000, null, 172800000, 172800000]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->scalar("172800000"), this->scalar("345600000")}); + this->Assert(MinElementWise, this->array("[null, null, null, null]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->scalar("null"), this->scalar("172800000")}); + this->Assert(MinElementWise, this->array("[86400000, null, 172800000, 172800000]"), + {this->array("[86400000, 172800000, 259200000, 345600000]"), + this->array("[172800000, null, 172800000, 172800000]")}); + + this->Assert(MinElementWise, this->array("[null, null, null, null]"), + {this->scalar("86400000"), this->array("[null, null, null, null]")}); + this->Assert( + MinElementWise, this->array("[null, null, null, null]"), + {this->scalar("null"), this->array("[86400000, 86400000, 86400000, 86400000]")}); + } else { + this->Assert(MinElementWise, this->scalar("0"), {this->scalar("0")}); + this->Assert(MinElementWise, this->scalar("0"), + {this->scalar("2"), this->scalar("0"), this->scalar("1")}); + this->Assert( + MinElementWise, this->scalar("0"), + {this->scalar("2"), this->scalar("0"), this->scalar("1"), this->scalar("null")}); + this->Assert(MinElementWise, this->scalar("1"), + {this->scalar("null"), this->scalar("null"), this->scalar("1"), + this->scalar("null")}); + + this->Assert(MinElementWise, (this->array("[]")), {this->array("[]")}); + this->Assert(MinElementWise, this->array("[1, 2, 3, null]"), + {this->array("[1, 2, 3, null]")}); + + this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), + {this->array("[1, 2, 3, 4]"), this->scalar("2")}); + this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), + {this->array("[1, null, 3, 4]"), this->scalar("2")}); + this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), + {this->array("[1, null, 3, 4]"), this->scalar("2"), this->scalar("4")}); + this->Assert( + MinElementWise, this->array("[1, 2, 2, 2]"), + {this->array("[1, null, 3, 4]"), this->scalar("null"), this->scalar("2")}); + + this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), + {this->array("[1, 2, 3, 4]"), this->array("[2, 2, 2, 2]")}); + this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), + {this->array("[1, 2, 3, 4]"), this->array("[2, null, 2, 2]")}); + this->Assert(MinElementWise, this->array("[1, 2, 2, 2]"), + {this->array("[1, null, 3, 4]"), this->array("[2, 2, 2, 2]")}); + + this->Assert(MinElementWise, this->array("[1, 2, null, 6]"), + {this->array("[1, 2, null, null]"), this->array("[4, null, null, 6]")}); + this->Assert(MinElementWise, this->array("[1, 2, null, 6]"), + {this->array("[4, null, null, 6]"), this->array("[1, 2, null, null]")}); + this->Assert(MinElementWise, this->array("[1, 2, 3, 4]"), + {this->array("[1, 2, 3, 4]"), this->array("[null, null, null, null]")}); + this->Assert(MinElementWise, this->array("[1, 2, 3, 4]"), + {this->array("[null, null, null, null]"), this->array("[1, 2, 3, 4]")}); + + this->Assert(MinElementWise, this->array("[1, 1, 1, 1]"), + {this->scalar("1"), this->array("[1, 2, 3, 4]")}); + this->Assert(MinElementWise, this->array("[1, 1, 1, 1]"), + {this->scalar("1"), this->array("[null, null, null, null]")}); + this->Assert(MinElementWise, this->array("[1, 1, 1, 1]"), + {this->scalar("null"), this->array("[1, 1, 1, 1]")}); + this->Assert(MinElementWise, this->array("[null, null, null, null]"), + {this->scalar("null"), this->array("[null, null, null, null]")}); + + // Test null handling + this->element_wise_aggregate_options_.skip_nulls = false; + this->AssertNullScalar(MinElementWise, {this->scalar("null"), this->scalar("null")}); + this->AssertNullScalar(MinElementWise, {this->scalar("0"), this->scalar("null")}); + + this->Assert(MinElementWise, this->array("[1, null, 2, 2]"), + {this->array("[1, null, 3, 4]"), this->scalar("2"), this->scalar("4")}); + this->Assert( + MinElementWise, this->array("[null, null, null, null]"), + {this->array("[1, null, 3, 4]"), this->scalar("null"), this->scalar("2")}); + this->Assert(MinElementWise, this->array("[1, null, 2, 2]"), + {this->array("[1, 2, 3, 4]"), this->array("[2, null, 2, 2]")}); + + this->Assert(MinElementWise, this->array("[null, null, null, null]"), + {this->scalar("1"), this->array("[null, null, null, null]")}); + this->Assert(MinElementWise, this->array("[null, null, null, null]"), + {this->scalar("null"), this->array("[1, 1, 1, 1]")}); + } } TYPED_TEST(TestVarArgsCompareDecimal, MinElementWise) { @@ -1539,71 +1635,167 @@ TYPED_TEST(TestVarArgsCompareFixedSizeBinary, MinElementWise) { TYPED_TEST(TestVarArgsCompareNumeric, MaxElementWise) { this->AssertNullScalar(MaxElementWise, {}); this->AssertNullScalar(MaxElementWise, {this->scalar("null"), this->scalar("null")}); - this->Assert(MaxElementWise, this->scalar("0"), {this->scalar("0")}); - this->Assert(MaxElementWise, this->scalar("2"), - {this->scalar("2"), this->scalar("0"), this->scalar("1")}); - this->Assert( - MaxElementWise, this->scalar("2"), - {this->scalar("2"), this->scalar("0"), this->scalar("1"), this->scalar("null")}); - this->Assert(MaxElementWise, this->scalar("1"), - {this->scalar("null"), this->scalar("null"), this->scalar("1"), - this->scalar("null")}); - - this->Assert(MaxElementWise, (this->array("[]")), {this->array("[]")}); - this->Assert(MaxElementWise, this->array("[1, 2, 3, null]"), - {this->array("[1, 2, 3, null]")}); - - this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), - {this->array("[1, 2, 3, 4]"), this->scalar("2")}); - this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), - {this->array("[1, null, 3, 4]"), this->scalar("2")}); - this->Assert(MaxElementWise, this->array("[4, 4, 4, 4]"), - {this->array("[1, null, 3, 4]"), this->scalar("2"), this->scalar("4")}); - this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), - {this->array("[1, null, 3, 4]"), this->scalar("null"), this->scalar("2")}); - this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), - {this->array("[1, 2, 3, 4]"), this->array("[2, 2, 2, 2]")}); - this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), - {this->array("[1, 2, 3, 4]"), this->array("[2, null, 2, 2]")}); - this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), - {this->array("[1, null, 3, 4]"), this->array("[2, 2, 2, 2]")}); - - this->Assert(MaxElementWise, this->array("[4, 2, null, 6]"), - {this->array("[1, 2, null, null]"), this->array("[4, null, null, 6]")}); - this->Assert(MaxElementWise, this->array("[4, 2, null, 6]"), - {this->array("[4, null, null, 6]"), this->array("[1, 2, null, null]")}); - this->Assert(MaxElementWise, this->array("[1, 2, 3, 4]"), - {this->array("[1, 2, 3, 4]"), this->array("[null, null, null, null]")}); - this->Assert(MaxElementWise, this->array("[1, 2, 3, 4]"), - {this->array("[null, null, null, null]"), this->array("[1, 2, 3, 4]")}); - - this->Assert(MaxElementWise, this->array("[1, 2, 3, 4]"), - {this->scalar("1"), this->array("[1, 2, 3, 4]")}); - this->Assert(MaxElementWise, this->array("[1, 1, 1, 1]"), - {this->scalar("1"), this->array("[null, null, null, null]")}); - this->Assert(MaxElementWise, this->array("[1, 1, 1, 1]"), - {this->scalar("null"), this->array("[1, 1, 1, 1]")}); - this->Assert(MaxElementWise, this->array("[null, null, null, null]"), - {this->scalar("null"), this->array("[null, null, null, null]")}); - - // Test null handling - this->element_wise_aggregate_options_.skip_nulls = false; - this->AssertNullScalar(MaxElementWise, {this->scalar("null"), this->scalar("null")}); - this->AssertNullScalar(MaxElementWise, {this->scalar("0"), this->scalar("null")}); - - this->Assert(MaxElementWise, this->array("[4, null, 4, 4]"), - {this->array("[1, null, 3, 4]"), this->scalar("2"), this->scalar("4")}); - this->Assert(MaxElementWise, this->array("[null, null, null, null]"), - {this->array("[1, null, 3, 4]"), this->scalar("null"), this->scalar("2")}); - this->Assert(MaxElementWise, this->array("[2, null, 3, 4]"), - {this->array("[1, 2, 3, 4]"), this->array("[2, null, 2, 2]")}); - - this->Assert(MaxElementWise, this->array("[null, null, null, null]"), - {this->scalar("1"), this->array("[null, null, null, null]")}); - this->Assert(MaxElementWise, this->array("[null, null, null, null]"), - {this->scalar("null"), this->array("[1, 1, 1, 1]")}); + if (std::is_same::value) { + this->Assert( + MaxElementWise, this->scalar("172800000"), + {this->scalar("172800000"), this->scalar("0"), this->scalar("86400000")}); + this->Assert(MaxElementWise, this->scalar("172800000"), + {this->scalar("172800000"), this->scalar("0"), this->scalar("86400000"), + this->scalar("null")}); + this->Assert(MaxElementWise, this->scalar("86400000"), + {this->scalar("null"), this->scalar("null"), this->scalar("86400000"), + this->scalar("null")}); + + this->Assert(MaxElementWise, (this->array("[]")), {this->array("[]")}); + this->Assert(MaxElementWise, this->array("[86400000, 172800000, 259200000, null]"), + {this->array("[86400000, 172800000, 259200000, null]")}); + + this->Assert(MaxElementWise, + this->array("[172800000, 172800000, 259200000, 345600000]"), + {this->array("[86400000, 172800000, 259200000, 345600000]"), + this->scalar("172800000")}); + this->Assert(MaxElementWise, + this->array("[172800000, 172800000, 259200000, 345600000]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->scalar("172800000")}); + this->Assert(MaxElementWise, + this->array("[345600000, 345600000, 345600000, 345600000]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->scalar("172800000"), this->scalar("345600000")}); + this->Assert(MaxElementWise, + this->array("[172800000, 172800000, 259200000, 345600000]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->scalar("null"), this->scalar("172800000")}); + + this->Assert(MaxElementWise, + this->array("[172800000, 172800000, 259200000, 345600000]"), + {this->array("[86400000, 172800000, 259200000, 345600000]"), + this->array("[172800000, 172800000, 172800000, 172800000]")}); + this->Assert(MaxElementWise, + this->array("[172800000, 172800000, 259200000, 345600000]"), + {this->array("[86400000, 172800000, 259200000, 345600000]"), + this->array("[172800000, null, 172800000, 172800000]")}); + this->Assert(MaxElementWise, + this->array("[172800000, 172800000, 259200000, 345600000]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->array("[172800000, 172800000, 172800000, 172800000]")}); + + this->Assert(MaxElementWise, this->array("[345600000, 172800000, null, 518400000]"), + {this->array("[86400000, 172800000, null, null]"), + this->array("[345600000, null, null, 518400000]")}); + this->Assert(MaxElementWise, this->array("[345600000, 172800000, null, 518400000]"), + {this->array("[345600000, null, null, 518400000]"), + this->array("[86400000, 172800000, null, null]")}); + this->Assert(MaxElementWise, + this->array("[86400000, 172800000, 259200000, 345600000]"), + {this->array("[86400000, 172800000, 259200000, 345600000]"), + this->array("[null, null, null, null]")}); + this->Assert(MaxElementWise, + this->array("[86400000, 172800000, 259200000, 345600000]"), + {this->array("[null, null, null, null]"), + this->array("[86400000, 172800000, 259200000, 345600000]")}); + + this->Assert(MaxElementWise, + this->array("[86400000, 172800000, 259200000, 345600000]"), + {this->scalar("86400000"), + this->array("[86400000, 172800000, 259200000, 345600000]")}); + this->Assert(MaxElementWise, this->array("[86400000, 86400000, 86400000, 86400000]"), + {this->scalar("86400000"), this->array("[null, null, null, null]")}); + this->Assert( + MaxElementWise, this->array("[86400000, 86400000, 86400000, 86400000]"), + {this->scalar("null"), this->array("[86400000, 86400000, 86400000, 86400000]")}); + this->Assert(MaxElementWise, this->array("[null, null, null, null]"), + {this->scalar("null"), this->array("[null, null, null, null]")}); + + // Test null handling + this->element_wise_aggregate_options_.skip_nulls = false; + this->AssertNullScalar(MaxElementWise, {this->scalar("null"), this->scalar("null")}); + this->AssertNullScalar(MaxElementWise, {this->scalar("0"), this->scalar("null")}); + + this->Assert(MaxElementWise, this->array("[345600000, null, 345600000, 345600000]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->scalar("172800000"), this->scalar("345600000")}); + this->Assert(MaxElementWise, this->array("[null, null, null, null]"), + {this->array("[86400000, null, 259200000, 345600000]"), + this->scalar("null"), this->scalar("172800000")}); + this->Assert(MaxElementWise, this->array("[172800000, null, 259200000, 345600000]"), + {this->array("[86400000, 172800000, 259200000, 345600000]"), + this->array("[172800000, null, 172800000, 172800000]")}); + + this->Assert(MaxElementWise, this->array("[null, null, null, null]"), + {this->scalar("86400000"), this->array("[null, null, null, null]")}); + this->Assert( + MaxElementWise, this->array("[null, null, null, null]"), + {this->scalar("null"), this->array("[86400000, 86400000, 86400000, 86400000]")}); + } else { + this->Assert(MaxElementWise, this->scalar("2"), + {this->scalar("2"), this->scalar("0"), this->scalar("1")}); + this->Assert( + MaxElementWise, this->scalar("2"), + {this->scalar("2"), this->scalar("0"), this->scalar("1"), this->scalar("null")}); + this->Assert(MaxElementWise, this->scalar("1"), + {this->scalar("null"), this->scalar("null"), this->scalar("1"), + this->scalar("null")}); + + this->Assert(MaxElementWise, (this->array("[]")), {this->array("[]")}); + this->Assert(MaxElementWise, this->array("[1, 2, 3, null]"), + {this->array("[1, 2, 3, null]")}); + + this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), + {this->array("[1, 2, 3, 4]"), this->scalar("2")}); + this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), + {this->array("[1, null, 3, 4]"), this->scalar("2")}); + this->Assert(MaxElementWise, this->array("[4, 4, 4, 4]"), + {this->array("[1, null, 3, 4]"), this->scalar("2"), this->scalar("4")}); + this->Assert( + MaxElementWise, this->array("[2, 2, 3, 4]"), + {this->array("[1, null, 3, 4]"), this->scalar("null"), this->scalar("2")}); + + this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), + {this->array("[1, 2, 3, 4]"), this->array("[2, 2, 2, 2]")}); + this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), + {this->array("[1, 2, 3, 4]"), this->array("[2, null, 2, 2]")}); + this->Assert(MaxElementWise, this->array("[2, 2, 3, 4]"), + {this->array("[1, null, 3, 4]"), this->array("[2, 2, 2, 2]")}); + + this->Assert(MaxElementWise, this->array("[4, 2, null, 6]"), + {this->array("[1, 2, null, null]"), this->array("[4, null, null, 6]")}); + this->Assert(MaxElementWise, this->array("[4, 2, null, 6]"), + {this->array("[4, null, null, 6]"), this->array("[1, 2, null, null]")}); + this->Assert(MaxElementWise, this->array("[1, 2, 3, 4]"), + {this->array("[1, 2, 3, 4]"), this->array("[null, null, null, null]")}); + this->Assert(MaxElementWise, this->array("[1, 2, 3, 4]"), + {this->array("[null, null, null, null]"), this->array("[1, 2, 3, 4]")}); + + this->Assert(MaxElementWise, this->array("[1, 2, 3, 4]"), + {this->scalar("1"), this->array("[1, 2, 3, 4]")}); + this->Assert(MaxElementWise, this->array("[1, 1, 1, 1]"), + {this->scalar("1"), this->array("[null, null, null, null]")}); + this->Assert(MaxElementWise, this->array("[1, 1, 1, 1]"), + {this->scalar("null"), this->array("[1, 1, 1, 1]")}); + this->Assert(MaxElementWise, this->array("[null, null, null, null]"), + {this->scalar("null"), this->array("[null, null, null, null]")}); + + // Test null handling + this->element_wise_aggregate_options_.skip_nulls = false; + this->AssertNullScalar(MaxElementWise, {this->scalar("null"), this->scalar("null")}); + this->AssertNullScalar(MaxElementWise, {this->scalar("0"), this->scalar("null")}); + + this->Assert(MaxElementWise, this->array("[4, null, 4, 4]"), + {this->array("[1, null, 3, 4]"), this->scalar("2"), this->scalar("4")}); + this->Assert( + MaxElementWise, this->array("[null, null, null, null]"), + {this->array("[1, null, 3, 4]"), this->scalar("null"), this->scalar("2")}); + this->Assert(MaxElementWise, this->array("[2, null, 3, 4]"), + {this->array("[1, 2, 3, 4]"), this->array("[2, null, 2, 2]")}); + + this->Assert(MaxElementWise, this->array("[null, null, null, null]"), + {this->scalar("1"), this->array("[null, null, null, null]")}); + this->Assert(MaxElementWise, this->array("[null, null, null, null]"), + {this->scalar("null"), this->array("[1, 1, 1, 1]")}); + } } TYPED_TEST(TestVarArgsCompareDecimal, MaxElementWise) { diff --git a/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc b/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc index 9d874238b5a..81d1341990b 100644 --- a/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc @@ -185,45 +185,95 @@ void CheckWithDifferentShapes(const std::shared_ptr& cond, TYPED_TEST(TestIfElsePrimitive, IfElseFixedSize) { auto type = default_type_instance(); - CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), - ArrayFromJSON(type, "[1, 2, 3, 4]"), - ArrayFromJSON(type, "[5, 6, 7, 8]"), - ArrayFromJSON(type, "[1, 2, 3, 8]")); - - CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), - ArrayFromJSON(type, "[1, 2, 3, 4]"), - ArrayFromJSON(type, "[5, 6, 7, null]"), - ArrayFromJSON(type, "[1, 2, 3, null]")); - - CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), - ArrayFromJSON(type, "[1, 2, null, 4]"), - ArrayFromJSON(type, "[5, 6, 7, null]"), - ArrayFromJSON(type, "[1, 2, null, null]")); - - CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), - ArrayFromJSON(type, "[1, 2, null, 4]"), - ArrayFromJSON(type, "[5, 6, 7, 8]"), - ArrayFromJSON(type, "[1, 2, null, 8]")); - - CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[null, true, true, false]"), - ArrayFromJSON(type, "[1, 2, null, 4]"), - ArrayFromJSON(type, "[5, 6, 7, 8]"), - ArrayFromJSON(type, "[null, 2, null, 8]")); + if (std::is_same::value) { + CheckWithDifferentShapes( + ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(type, "[86400000, 172800000, 259200000, 345600000]"), + ArrayFromJSON(type, "[432000000, 518400000, 604800000, 691200000]"), + ArrayFromJSON(type, "[86400000, 172800000, 259200000, 691200000]")); + + CheckWithDifferentShapes( + ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(type, "[86400000, 172800000, 259200000, 345600000]"), + ArrayFromJSON(type, "[432000000, 518400000, 604800000, null]"), + ArrayFromJSON(type, "[86400000, 172800000, 259200000, null]")); + + CheckWithDifferentShapes( + ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(type, "[86400000, 172800000, null, 345600000]"), + ArrayFromJSON(type, "[432000000, 518400000, 604800000, null]"), + ArrayFromJSON(type, "[86400000, 172800000, null, null]")); + + CheckWithDifferentShapes( + ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(type, "[86400000, 172800000, null, 345600000]"), + ArrayFromJSON(type, "[432000000, 518400000, 604800000, 691200000]"), + ArrayFromJSON(type, "[86400000, 172800000, null, 691200000]")); + + CheckWithDifferentShapes( + ArrayFromJSON(boolean(), "[null, true, true, false]"), + ArrayFromJSON(type, "[86400000, 172800000, null, 345600000]"), + ArrayFromJSON(type, "[432000000, 518400000, 604800000, 691200000]"), + ArrayFromJSON(type, "[null, 172800000, null, 691200000]")); + + CheckWithDifferentShapes( + ArrayFromJSON(boolean(), "[null, true, true, false]"), + ArrayFromJSON(type, "[86400000, 172800000, null, 345600000]"), + ArrayFromJSON(type, "[432000000, 518400000, 604800000, null]"), + ArrayFromJSON(type, "[null, 172800000, null, null]")); + + CheckWithDifferentShapes( + ArrayFromJSON(boolean(), "[null, true, true, false]"), + ArrayFromJSON(type, "[86400000, 172800000, 259200000, 345600000]"), + ArrayFromJSON(type, "[432000000, 518400000, 604800000, null]"), + ArrayFromJSON(type, "[null, 172800000, 259200000, null]")); + + CheckWithDifferentShapes( + ArrayFromJSON(boolean(), "[null, true, true, false]"), + ArrayFromJSON(type, "[86400000, 172800000, 259200000, 345600000]"), + ArrayFromJSON(type, "[432000000, 518400000, 604800000, 691200000]"), + ArrayFromJSON(type, "[null, 172800000, 259200000, 691200000]")); + } else { + CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(type, "[1, 2, 3, 4]"), + ArrayFromJSON(type, "[5, 6, 7, 8]"), + ArrayFromJSON(type, "[1, 2, 3, 8]")); - CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[null, true, true, false]"), - ArrayFromJSON(type, "[1, 2, null, 4]"), - ArrayFromJSON(type, "[5, 6, 7, null]"), - ArrayFromJSON(type, "[null, 2, null, null]")); + CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(type, "[1, 2, 3, 4]"), + ArrayFromJSON(type, "[5, 6, 7, null]"), + ArrayFromJSON(type, "[1, 2, 3, null]")); - CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[null, true, true, false]"), - ArrayFromJSON(type, "[1, 2, 3, 4]"), - ArrayFromJSON(type, "[5, 6, 7, null]"), - ArrayFromJSON(type, "[null, 2, 3, null]")); + CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(type, "[1, 2, null, 4]"), + ArrayFromJSON(type, "[5, 6, 7, null]"), + ArrayFromJSON(type, "[1, 2, null, null]")); - CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[null, true, true, false]"), - ArrayFromJSON(type, "[1, 2, 3, 4]"), - ArrayFromJSON(type, "[5, 6, 7, 8]"), - ArrayFromJSON(type, "[null, 2, 3, 8]")); + CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(type, "[1, 2, null, 4]"), + ArrayFromJSON(type, "[5, 6, 7, 8]"), + ArrayFromJSON(type, "[1, 2, null, 8]")); + + CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[null, true, true, false]"), + ArrayFromJSON(type, "[1, 2, null, 4]"), + ArrayFromJSON(type, "[5, 6, 7, 8]"), + ArrayFromJSON(type, "[null, 2, null, 8]")); + + CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[null, true, true, false]"), + ArrayFromJSON(type, "[1, 2, null, 4]"), + ArrayFromJSON(type, "[5, 6, 7, null]"), + ArrayFromJSON(type, "[null, 2, null, null]")); + + CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[null, true, true, false]"), + ArrayFromJSON(type, "[1, 2, 3, 4]"), + ArrayFromJSON(type, "[5, 6, 7, null]"), + ArrayFromJSON(type, "[null, 2, 3, null]")); + + CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[null, true, true, false]"), + ArrayFromJSON(type, "[1, 2, 3, 4]"), + ArrayFromJSON(type, "[5, 6, 7, 8]"), + ArrayFromJSON(type, "[null, 2, 3, 8]")); + } } TEST_F(TestIfElseKernel, IfElseBoolean) { @@ -332,10 +382,18 @@ TEST_F(TestIfElseKernel, TimestampTypes) { TEST_F(TestIfElseKernel, TemporalTypes) { for (const auto& ty : TemporalTypes()) { - CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), - ArrayFromJSON(ty, "[1, 2, 3, 4]"), - ArrayFromJSON(ty, "[5, 6, 7, 8]"), - ArrayFromJSON(ty, "[1, 2, 3, 8]")); + if (ty->name() == "date64") { + CheckWithDifferentShapes( + ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(ty, "[86400000, 172800000, 259200000, 4]"), + ArrayFromJSON(ty, "[5, 6, 7, 691200000]"), + ArrayFromJSON(ty, "[86400000, 172800000, 259200000, 691200000]")); + } else { + CheckWithDifferentShapes(ArrayFromJSON(boolean(), "[true, true, true, false]"), + ArrayFromJSON(ty, "[1, 2, 3, 4]"), + ArrayFromJSON(ty, "[5, 6, 7, 8]"), + ArrayFromJSON(ty, "[1, 2, 3, 8]")); + } } } @@ -978,89 +1036,196 @@ TYPED_TEST(TestCaseWhenNumeric, FixedSize) { auto cond1 = ArrayFromJSON(boolean(), "[true, true, null, null]"); auto cond2 = ArrayFromJSON(boolean(), "[true, false, true, null]"); auto scalar_null = ScalarFromJSON(type, "null"); - auto scalar1 = ScalarFromJSON(type, "1"); - auto scalar2 = ScalarFromJSON(type, "2"); auto values_null = ArrayFromJSON(type, "[null, null, null, null]"); - auto values1 = ArrayFromJSON(type, "[3, null, 5, 6]"); - auto values2 = ArrayFromJSON(type, "[7, 8, null, 10]"); - CheckScalar("case_when", {MakeStruct({}), values1}, values1); - CheckScalar("case_when", {MakeStruct({}), values_null}, values_null); + if (std::is_same::value) { + auto scalar1 = ScalarFromJSON(type, "86400000"); + auto scalar2 = ScalarFromJSON(type, "172800000"); + auto values1 = ArrayFromJSON(type, "[259200000, null, 432000000, 518400000]"); + auto values2 = ArrayFromJSON(type, "[604800000, 691200000, null, 864000000]"); - CheckScalar("case_when", {MakeStruct({cond_true}), scalar1, values1}, - *MakeArrayFromScalar(*scalar1, 4)); - CheckScalar("case_when", {MakeStruct({cond_false}), scalar1, values1}, values1); + CheckScalar("case_when", {MakeStruct({}), values1}, values1); + CheckScalar("case_when", {MakeStruct({}), values_null}, values_null); - CheckScalar("case_when", {MakeStruct({cond_true}), values1}, values1); - CheckScalar("case_when", {MakeStruct({cond_false}), values1}, values_null); - CheckScalar("case_when", {MakeStruct({cond_null}), values1}, values_null); - CheckScalar("case_when", {MakeStruct({cond_true}), values1, values2}, values1); - CheckScalar("case_when", {MakeStruct({cond_false}), values1, values2}, values2); - CheckScalar("case_when", {MakeStruct({cond_null}), values1, values2}, values2); + CheckScalar("case_when", {MakeStruct({cond_true}), scalar1, values1}, + *MakeArrayFromScalar(*scalar1, 4)); + CheckScalar("case_when", {MakeStruct({cond_false}), scalar1, values1}, values1); - CheckScalar("case_when", {MakeStruct({cond_true, cond_true}), values1, values2}, - values1); - CheckScalar("case_when", {MakeStruct({cond_false, cond_false}), values1, values2}, - values_null); - CheckScalar("case_when", {MakeStruct({cond_true, cond_false}), values1, values2}, - values1); - CheckScalar("case_when", {MakeStruct({cond_false, cond_true}), values1, values2}, - values2); - CheckScalar("case_when", {MakeStruct({cond_null, cond_true}), values1, values2}, - values2); - CheckScalar("case_when", - {MakeStruct({cond_false, cond_false}), values1, values2, values2}, values2); + CheckScalar("case_when", {MakeStruct({cond_true}), values1}, values1); + CheckScalar("case_when", {MakeStruct({cond_false}), values1}, values_null); + CheckScalar("case_when", {MakeStruct({cond_null}), values1}, values_null); + CheckScalar("case_when", {MakeStruct({cond_true}), values1, values2}, values1); + CheckScalar("case_when", {MakeStruct({cond_false}), values1, values2}, values2); + CheckScalar("case_when", {MakeStruct({cond_null}), values1, values2}, values2); - CheckScalar("case_when", {MakeStruct({cond1, cond2}), scalar1, scalar2}, - ArrayFromJSON(type, "[1, 1, 2, null]")); - CheckScalar("case_when", {MakeStruct({cond1}), scalar_null}, values_null); - CheckScalar("case_when", {MakeStruct({cond1}), scalar_null, scalar1}, - ArrayFromJSON(type, "[null, null, 1, 1]")); - CheckScalar("case_when", {MakeStruct({cond1, cond2}), scalar1, scalar2, scalar1}, - ArrayFromJSON(type, "[1, 1, 2, 1]")); + CheckScalar("case_when", {MakeStruct({cond_true, cond_true}), values1, values2}, + values1); + CheckScalar("case_when", {MakeStruct({cond_false, cond_false}), values1, values2}, + values_null); + CheckScalar("case_when", {MakeStruct({cond_true, cond_false}), values1, values2}, + values1); + CheckScalar("case_when", {MakeStruct({cond_false, cond_true}), values1, values2}, + values2); + CheckScalar("case_when", {MakeStruct({cond_null, cond_true}), values1, values2}, + values2); + CheckScalar("case_when", + {MakeStruct({cond_false, cond_false}), values1, values2, values2}, + values2); - CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2}, - ArrayFromJSON(type, "[3, null, null, null]")); - CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2, values1}, - ArrayFromJSON(type, "[3, null, null, 6]")); - CheckScalar("case_when", {MakeStruct({cond1, cond2}), values_null, values2, values1}, - ArrayFromJSON(type, "[null, null, null, 6]")); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), scalar1, scalar2}, + ArrayFromJSON(type, "[86400000, 86400000, 172800000, null]")); + CheckScalar("case_when", {MakeStruct({cond1}), scalar_null}, values_null); + CheckScalar("case_when", {MakeStruct({cond1}), scalar_null, scalar1}, + ArrayFromJSON(type, "[null, null, 86400000, 86400000]")); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), scalar1, scalar2, scalar1}, + ArrayFromJSON(type, "[86400000, 86400000, 172800000, 86400000]")); - CheckScalar( - "case_when", - {MakeStruct( - {ArrayFromJSON(boolean(), - "[true, true, true, false, false, false, null, null, null]"), - ArrayFromJSON(boolean(), - "[true, false, null, true, false, null, true, false, null]")}), - ArrayFromJSON(type, "[10, 11, 12, 13, 14, 15, 16, 17, 18]"), - ArrayFromJSON(type, "[20, 21, 22, 23, 24, 25, 26, 27, 28]")}, - ArrayFromJSON(type, "[10, 11, 12, 23, null, null, 26, null, null]")); - CheckScalar( - "case_when", - {MakeStruct( - {ArrayFromJSON(boolean(), - "[true, true, true, false, false, false, null, null, null]"), - ArrayFromJSON(boolean(), - "[true, false, null, true, false, null, true, false, null]")}), - ArrayFromJSON(type, "[10, 11, 12, 13, 14, 15, 16, 17, 18]"), - - ArrayFromJSON(type, "[20, 21, 22, 23, 24, 25, 26, 27, 28]"), - ArrayFromJSON(type, "[30, 31, 32, 33, 34, null, 36, 37, null]")}, - ArrayFromJSON(type, "[10, 11, 12, 23, 34, null, 26, 37, null]")); - - // Error cases - EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, ::testing::HasSubstr("cond struct must not be null"), - CallFunction( - "case_when", - {Datum(std::make_shared(struct_({field("", boolean())}))), - Datum(scalar1)})); - EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, ::testing::HasSubstr("cond struct must not have top-level nulls"), - CallFunction( - "case_when", - {Datum(*MakeArrayOfNull(struct_({field("", boolean())}), 4)), Datum(values1)})); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2}, + ArrayFromJSON(type, "[259200000, null, null, null]")); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2, values1}, + ArrayFromJSON(type, "[259200000, null, null, 518400000]")); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values_null, values2, values1}, + ArrayFromJSON(type, "[null, null, null, 518400000]")); + + CheckScalar( + "case_when", + {MakeStruct( + {ArrayFromJSON(boolean(), + "[true, true, true, false, false, false, null, null, null]"), + ArrayFromJSON( + boolean(), + "[true, false, null, true, false, null, true, false, null]")}), + ArrayFromJSON(type, + "[864000000, 950400000, 1036800000, 1123200000, 1209600000, " + "1296000000, 1382400000, 1468800000, 1555200000]"), + ArrayFromJSON(type, + "[1728000000, 1814400000, 1900800000, 1987200000, 2073600000, " + "2160000000, 2246400000, 2332800000, 2419200000]")}, + ArrayFromJSON(type, + "[864000000, 950400000, 1036800000, 1987200000, null, null, " + "2246400000, null, null]")); + CheckScalar( + "case_when", + {MakeStruct( + {ArrayFromJSON(boolean(), + "[true, true, true, false, false, false, null, null, null]"), + ArrayFromJSON( + boolean(), + "[true, false, null, true, false, null, true, false, null]")}), + ArrayFromJSON(type, + "[864000000, 950400000, 1036800000, 1123200000, 1209600000, " + "1296000000, 1382400000, 1468800000, 1555200000]"), + + ArrayFromJSON(type, + "[1728000000, 1814400000, 1900800000, 1987200000, 2073600000, " + "2160000000, 2246400000, 2332800000, 2419200000]"), + ArrayFromJSON(type, + "[2592000000, 2678400000, 2764800000, 2851200000, 2937600000, " + "null, 3110400000, 3196800000, null]")}, + ArrayFromJSON(type, + "[864000000, 950400000, 1036800000, 1987200000, 2937600000, null, " + "2246400000, 3196800000, null]")); + + // Error cases + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("cond struct must not be null"), + CallFunction( + "case_when", + {Datum(std::make_shared(struct_({field("", boolean())}))), + Datum(scalar1)})); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("cond struct must not have top-level nulls"), + CallFunction("case_when", + {Datum(*MakeArrayOfNull(struct_({field("", boolean())}), 4)), + Datum(values1)})); + + } else { + auto scalar1 = ScalarFromJSON(type, "1"); + auto scalar2 = ScalarFromJSON(type, "2"); + auto values1 = ArrayFromJSON(type, "[3, null, 5, 6]"); + auto values2 = ArrayFromJSON(type, "[604800000, 8, null, 10]"); + + CheckScalar("case_when", {MakeStruct({}), values1}, values1); + CheckScalar("case_when", {MakeStruct({}), values_null}, values_null); + + CheckScalar("case_when", {MakeStruct({cond_true}), scalar1, values1}, + *MakeArrayFromScalar(*scalar1, 4)); + CheckScalar("case_when", {MakeStruct({cond_false}), scalar1, values1}, values1); + + CheckScalar("case_when", {MakeStruct({cond_true}), values1}, values1); + CheckScalar("case_when", {MakeStruct({cond_false}), values1}, values_null); + CheckScalar("case_when", {MakeStruct({cond_null}), values1}, values_null); + CheckScalar("case_when", {MakeStruct({cond_true}), values1, values2}, values1); + CheckScalar("case_when", {MakeStruct({cond_false}), values1, values2}, values2); + CheckScalar("case_when", {MakeStruct({cond_null}), values1, values2}, values2); + + CheckScalar("case_when", {MakeStruct({cond_true, cond_true}), values1, values2}, + values1); + CheckScalar("case_when", {MakeStruct({cond_false, cond_false}), values1, values2}, + values_null); + CheckScalar("case_when", {MakeStruct({cond_true, cond_false}), values1, values2}, + values1); + CheckScalar("case_when", {MakeStruct({cond_false, cond_true}), values1, values2}, + values2); + CheckScalar("case_when", {MakeStruct({cond_null, cond_true}), values1, values2}, + values2); + CheckScalar("case_when", + {MakeStruct({cond_false, cond_false}), values1, values2, values2}, + values2); + + CheckScalar("case_when", {MakeStruct({cond1, cond2}), scalar1, scalar2}, + ArrayFromJSON(type, "[1, 1, 2, null]")); + CheckScalar("case_when", {MakeStruct({cond1}), scalar_null}, values_null); + CheckScalar("case_when", {MakeStruct({cond1}), scalar_null, scalar1}, + ArrayFromJSON(type, "[null, null, 1, 1]")); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), scalar1, scalar2, scalar1}, + ArrayFromJSON(type, "[1, 1, 2, 1]")); + + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2}, + ArrayFromJSON(type, "[3, null, null, null]")); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2, values1}, + ArrayFromJSON(type, "[3, null, null, 6]")); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values_null, values2, values1}, + ArrayFromJSON(type, "[null, null, null, 6]")); + + CheckScalar( + "case_when", + {MakeStruct( + {ArrayFromJSON(boolean(), + "[true, true, true, false, false, false, null, null, null]"), + ArrayFromJSON( + boolean(), + "[true, false, null, true, false, null, true, false, null]")}), + ArrayFromJSON(type, "[10, 11, 12, 13, 14, 15, 16, 17, 18]"), + ArrayFromJSON(type, "[20, 21, 22, 23, 24, 25, 26, 27, 28]")}, + ArrayFromJSON(type, "[10, 11, 12, 23, null, null, 26, null, null]")); + CheckScalar( + "case_when", + {MakeStruct( + {ArrayFromJSON(boolean(), + "[true, true, true, false, false, false, null, null, null]"), + ArrayFromJSON( + boolean(), + "[true, false, null, true, false, null, true, false, null]")}), + ArrayFromJSON(type, "[10, 11, 12, 13, 14, 15, 16, 17, 18]"), + + ArrayFromJSON(type, "[20, 21, 22, 23, 24, 25, 26, 27, 28]"), + ArrayFromJSON(type, "[30, 31, 32, 33, 34, null, 36, 37, null]")}, + ArrayFromJSON(type, "[10, 11, 12, 23, 34, null, 26, 37, null]")); + + // Error cases + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("cond struct must not be null"), + CallFunction( + "case_when", + {Datum(std::make_shared(struct_({field("", boolean())}))), + Datum(scalar1)})); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, ::testing::HasSubstr("cond struct must not have top-level nulls"), + CallFunction("case_when", + {Datum(*MakeArrayOfNull(struct_({field("", boolean())}), 4)), + Datum(values1)})); + } } TYPED_TEST(TestCaseWhenNumeric, ListOfType) { @@ -2938,25 +3103,49 @@ TYPED_TEST_SUITE(TestChooseBinary, BaseBinaryArrowTypes); TYPED_TEST(TestChooseNumeric, FixedSize) { auto type = default_type_instance(); auto indices1 = ArrayFromJSON(int64(), "[0, 1, 0, 1, null]"); - auto values1 = ArrayFromJSON(type, "[10, 11, null, null, 14]"); - auto values2 = ArrayFromJSON(type, "[20, 21, null, null, 24]"); - auto nulls = ArrayFromJSON(type, "[null, null, null, null, null]"); - CheckScalar("choose", {indices1, values1, values2}, - ArrayFromJSON(type, "[10, 21, null, null, null]")); - CheckScalar("choose", {indices1, ScalarFromJSON(type, "1"), values1}, - ArrayFromJSON(type, "[1, 11, 1, null, null]")); - // Mixed scalar and array (note CheckScalar checks all-scalar cases for us) - CheckScalar("choose", {ScalarFromJSON(int64(), "0"), values1, values2}, values1); - CheckScalar("choose", {ScalarFromJSON(int64(), "1"), values1, values2}, values2); - CheckScalar("choose", {ScalarFromJSON(int64(), "null"), values1, values2}, nulls); - auto scalar1 = ScalarFromJSON(type, "42"); - CheckScalar("choose", {ScalarFromJSON(int64(), "0"), scalar1, values2}, - *MakeArrayFromScalar(*scalar1, 5)); - CheckScalar("choose", {ScalarFromJSON(int64(), "1"), scalar1, values2}, values2); - CheckScalar("choose", {ScalarFromJSON(int64(), "null"), values1, values2}, nulls); auto scalar_null = ScalarFromJSON(type, "null"); - CheckScalar("choose", {ScalarFromJSON(int64(), "0"), scalar_null, values2}, - *MakeArrayOfNull(type, 5)); + auto nulls = ArrayFromJSON(type, "[null, null, null, null, null]"); + + if (std::is_same::value) { + auto values1 = ArrayFromJSON(type, "[864000000, 950400000, null, null, 1209600000]"); + auto values2 = + ArrayFromJSON(type, "[1728000000, 1814400000, null, null, 2073600000]"); + auto scalar1 = ScalarFromJSON(type, "3628800000"); + + CheckScalar("choose", {indices1, values1, values2}, + ArrayFromJSON(type, "[864000000, 1814400000, null, null, null]")); + CheckScalar("choose", {indices1, ScalarFromJSON(type, "864000000"), values1}, + ArrayFromJSON(type, "[864000000, 950400000, 864000000, null, null]")); + // Mixed scalar and array (note CheckScalar checks all-scalar cases for us) + CheckScalar("choose", {ScalarFromJSON(int64(), "0"), values1, values2}, values1); + CheckScalar("choose", {ScalarFromJSON(int64(), "1"), values1, values2}, values2); + CheckScalar("choose", {ScalarFromJSON(int64(), "null"), values1, values2}, nulls); + CheckScalar("choose", {ScalarFromJSON(int64(), "0"), scalar1, values2}, + *MakeArrayFromScalar(*scalar1, 5)); + CheckScalar("choose", {ScalarFromJSON(int64(), "1"), scalar1, values2}, values2); + CheckScalar("choose", {ScalarFromJSON(int64(), "null"), values1, values2}, nulls); + CheckScalar("choose", {ScalarFromJSON(int64(), "0"), scalar_null, values2}, + *MakeArrayOfNull(type, 5)); + } else { + auto values1 = ArrayFromJSON(type, "[10, 11, null, null, 14]"); + auto values2 = ArrayFromJSON(type, "[20, 21, null, null, 24]"); + auto scalar1 = ScalarFromJSON(type, "42"); + + CheckScalar("choose", {indices1, values1, values2}, + ArrayFromJSON(type, "[10, 21, null, null, null]")); + CheckScalar("choose", {indices1, ScalarFromJSON(type, "1"), values1}, + ArrayFromJSON(type, "[1, 11, 1, null, null]")); + // Mixed scalar and array (note CheckScalar checks all-scalar cases for us) + CheckScalar("choose", {ScalarFromJSON(int64(), "0"), values1, values2}, values1); + CheckScalar("choose", {ScalarFromJSON(int64(), "1"), values1, values2}, values2); + CheckScalar("choose", {ScalarFromJSON(int64(), "null"), values1, values2}, nulls); + CheckScalar("choose", {ScalarFromJSON(int64(), "0"), scalar1, values2}, + *MakeArrayFromScalar(*scalar1, 5)); + CheckScalar("choose", {ScalarFromJSON(int64(), "1"), scalar1, values2}, values2); + CheckScalar("choose", {ScalarFromJSON(int64(), "null"), values1, values2}, nulls); + CheckScalar("choose", {ScalarFromJSON(int64(), "0"), scalar_null, values2}, + *MakeArrayOfNull(type, 5)); + } } TYPED_TEST(TestChooseBinary, Basics) { diff --git a/cpp/src/arrow/compute/kernels/vector_hash_test.cc b/cpp/src/arrow/compute/kernels/vector_hash_test.cc index d157b0d5fab..864f9484d73 100644 --- a/cpp/src/arrow/compute/kernels/vector_hash_test.cc +++ b/cpp/src/arrow/compute/kernels/vector_hash_test.cc @@ -161,42 +161,95 @@ TYPED_TEST_SUITE(TestHashKernelPrimitive, PrimitiveDictionaries); TYPED_TEST(TestHashKernelPrimitive, Unique) { using T = typename TypeParam::c_type; auto type = TypeTraits::type_singleton(); - CheckUnique(type, {2, 1, 2, 1}, {true, false, true, true}, {2, 0, 1}, - {1, 0, 1}); - CheckUnique(type, {2, 1, 3, 1}, {false, false, true, true}, {0, 3, 1}, - {0, 1, 1}); - // Sliced - CheckUnique(ArrayFromJSON(type, "[1, 2, null, 3, 2, null]")->Slice(1, 4), - ArrayFromJSON(type, "[2, null, 3]")); + if (type->name() == "date64") { + CheckUnique( + type, {172800000LL, 86400000LL, 172800000LL, 86400000LL}, + {true, false, true, true}, {172800000LL, 0, 86400000LL}, {1, 0, 1}); + CheckUnique( + type, {172800000LL, 86400000LL, 259200000LL, 86400000LL}, + {false, false, true, true}, {0, 259200000LL, 86400000LL}, {0, 1, 1}); + + // Sliced + CheckUnique( + ArrayFromJSON(type, "[86400000, 172800000, null, 259200000, 172800000, null]") + ->Slice(1, 4), + ArrayFromJSON(type, "[172800000, null, 259200000]")); + } else { + CheckUnique(type, {2, 1, 2, 1}, {true, false, true, true}, {2, 0, 1}, + {1, 0, 1}); + CheckUnique(type, {2, 1, 3, 1}, {false, false, true, true}, {0, 3, 1}, + {0, 1, 1}); + + // Sliced + CheckUnique(ArrayFromJSON(type, "[1, 2, null, 3, 2, null]")->Slice(1, 4), + ArrayFromJSON(type, "[2, null, 3]")); + } } TYPED_TEST(TestHashKernelPrimitive, ValueCounts) { using T = typename TypeParam::c_type; auto type = TypeTraits::type_singleton(); - CheckValueCounts(type, {2, 1, 2, 1, 2, 3, 4}, - {true, false, true, true, true, true, false}, - {2, 0, 1, 3}, {1, 0, 1, 1}, {3, 2, 1, 1}); - CheckValueCounts(type, {}, {}, {}, {}, {}); - CheckValueCountsNull(type); - // Sliced - CheckValueCounts(ArrayFromJSON(type, "[1, 2, null, 3, 2, null]")->Slice(1, 4), - ArrayFromJSON(type, "[2, null, 3]"), - ArrayFromJSON(int64(), "[2, 1, 1]")); + if (type->name() == "date64") { + CheckValueCounts( + type, + {172800000LL, 86400000LL, 172800000LL, 86400000LL, 172800000LL, 259200000LL, + 345600000LL}, + {true, false, true, true, true, true, false}, + {172800000LL, 0, 86400000LL, 259200000LL}, {1, 0, 1, 1}, {3, 2, 1, 1}); + CheckValueCounts(type, {}, {}, {}, {}, {}); + CheckValueCountsNull(type); + + // Sliced + CheckValueCounts( + ArrayFromJSON(type, "[86400000, 172800000, null, 259200000, 172800000, null]") + ->Slice(1, 4), + ArrayFromJSON(type, "[172800000, null, 259200000]"), + ArrayFromJSON(int64(), "[2, 1, 1]")); + } else { + CheckValueCounts(type, {2, 1, 2, 1, 2, 3, 4}, + {true, false, true, true, true, true, false}, + {2, 0, 1, 3}, {1, 0, 1, 1}, {3, 2, 1, 1}); + CheckValueCounts(type, {}, {}, {}, {}, {}); + CheckValueCountsNull(type); + + // Sliced + CheckValueCounts(ArrayFromJSON(type, "[1, 2, null, 3, 2, null]")->Slice(1, 4), + ArrayFromJSON(type, "[2, null, 3]"), + ArrayFromJSON(int64(), "[2, 1, 1]")); + } } TYPED_TEST(TestHashKernelPrimitive, DictEncode) { using T = typename TypeParam::c_type; auto type = TypeTraits::type_singleton(); - CheckDictEncode(type, {2, 1, 2, 1, 2, 3}, - {true, false, true, true, true, true}, {2, 1, 3}, - {1, 1, 1}, {0, 0, 0, 1, 0, 2}); - // Sliced - CheckDictEncode(ArrayFromJSON(type, "[2, 1, null, 4, 3, 1, 42]")->Slice(1, 5), - ArrayFromJSON(type, "[1, 4, 3]"), - ArrayFromJSON(int32(), "[0, null, 1, 2, 0]")); + if (type->name() == "date64") { + CheckDictEncode( + type, + {172800000LL, 86400000LL, 172800000LL, 86400000LL, 172800000LL, 345600000LL}, + {true, false, true, true, true, true}, {172800000LL, 86400000LL, 345600000LL}, + {1, 1, 1}, {0, 0, 0, 1, 0, 2}); + + // Sliced + CheckDictEncode( + ArrayFromJSON( + type, + "[172800000, 86400000, null, 345600000, 259200000, 86400000, 172800000]") + ->Slice(1, 5), + ArrayFromJSON(type, "[86400000, 345600000, 259200000]"), + ArrayFromJSON(int32(), "[0, null, 1, 2, 0]")); + } else { + CheckDictEncode(type, {2, 1, 2, 1, 2, 3}, + {true, false, true, true, true, true}, {2, 1, 3}, + {1, 1, 1}, {0, 0, 0, 1, 0, 2}); + + // Sliced + CheckDictEncode(ArrayFromJSON(type, "[2, 1, null, 4, 3, 1, 42]")->Slice(1, 5), + ArrayFromJSON(type, "[1, 4, 3]"), + ArrayFromJSON(int32(), "[0, null, 1, 2, 0]")); + } } TYPED_TEST(TestHashKernelPrimitive, ZeroChunks) { @@ -212,26 +265,42 @@ TYPED_TEST(TestHashKernelPrimitive, ZeroChunks) { TYPED_TEST(TestHashKernelPrimitive, PrimitiveResizeTable) { using T = typename TypeParam::c_type; + auto type = TypeTraits::type_singleton(); const int64_t kTotalValues = std::min(INT16_MAX, 1UL << sizeof(T) / 2); const int64_t kRepeats = 5; + constexpr int64_t kFullDayMillis = 1000 * 60 * 60 * 24; + const int64_t kTotalDate64Values = kFullDayMillis * kTotalValues; std::vector values; std::vector uniques; std::vector indices; std::vector counts; - for (int64_t i = 0; i < kTotalValues * kRepeats; i++) { - const auto val = static_cast(i % kTotalValues); - values.push_back(val); - if (i < kTotalValues) { - uniques.push_back(val); - counts.push_back(kRepeats); + if (type->name() == "date64") { + for (int64_t i = 0; i < kTotalDate64Values * kRepeats; i += kFullDayMillis) { + const auto val = static_cast(i % kTotalDate64Values); + values.push_back(val); + + if (i < kTotalDate64Values) { + uniques.push_back(val); + counts.push_back(kRepeats); + } + indices.push_back(static_cast(i % kTotalDate64Values / kFullDayMillis)); + } + } else { + for (int64_t i = 0; i < kTotalValues * kRepeats; i++) { + const auto val = static_cast(i % kTotalValues); + values.push_back(val); + + if (i < kTotalValues) { + uniques.push_back(val); + counts.push_back(kRepeats); + } + indices.push_back(static_cast(i % kTotalValues)); } - indices.push_back(static_cast(i % kTotalValues)); } - auto type = TypeTraits::type_singleton(); CheckUnique(type, values, {}, uniques, {}); CheckValueCounts(type, values, {}, uniques, {}, counts); CheckDictEncode(type, values, {}, uniques, {}, indices); diff --git a/cpp/src/arrow/compute/kernels/vector_replace_test.cc b/cpp/src/arrow/compute/kernels/vector_replace_test.cc index 8d3f0ae85b4..1ed1dbdaef0 100644 --- a/cpp/src/arrow/compute/kernels/vector_replace_test.cc +++ b/cpp/src/arrow/compute/kernels/vector_replace_test.cc @@ -232,91 +232,190 @@ TYPED_TEST(TestReplaceNumeric, ReplaceWithMask) { this->Assert(ReplaceWithMask, this->array("[]"), this->null_mask_scalar(), this->array("[]"), this->array("[]")); - this->Assert(ReplaceWithMask, this->array("[1]"), this->mask_scalar(false), - this->array("[]"), this->array("[1]")); - this->Assert(ReplaceWithMask, this->array("[1]"), this->mask_scalar(true), - this->array("[0]"), this->array("[0]")); - this->Assert(ReplaceWithMask, this->array("[1]"), this->mask_scalar(true), - this->array("[2, 0]"), this->array("[2]")); - this->Assert(ReplaceWithMask, this->array("[1]"), this->null_mask_scalar(), - this->array("[]"), this->array("[null]")); - - this->Assert(ReplaceWithMask, this->array("[0, 0]"), this->mask_scalar(false), - this->scalar("1"), this->array("[0, 0]")); - this->Assert(ReplaceWithMask, this->array("[0, 0]"), this->mask_scalar(true), - this->scalar("1"), this->array("[1, 1]")); - this->Assert(ReplaceWithMask, this->array("[0, 0]"), this->mask_scalar(true), - this->scalar("null"), this->array("[null, null]")); - - this->Assert(ReplaceWithMask, this->array("[]"), this->mask("[]"), this->array("[]"), - this->array("[]")); - this->Assert(ReplaceWithMask, this->array("[0, 1, 2, 3]"), - this->mask("[false, false, false, false]"), this->array("[]"), - this->array("[0, 1, 2, 3]")); - this->Assert(ReplaceWithMask, this->array("[0, 1, 2, 3]"), - this->mask("[true, true, true, true]"), this->array("[10, 11, 12, 13]"), - this->array("[10, 11, 12, 13]")); - this->Assert(ReplaceWithMask, this->array("[0, 1, 2, 3]"), - this->mask("[null, null, null, null]"), this->array("[]"), - this->array("[null, null, null, null]")); - this->Assert(ReplaceWithMask, this->array("[0, 1, 2, null]"), - this->mask("[false, false, false, false]"), this->array("[]"), - this->array("[0, 1, 2, null]")); - this->Assert(ReplaceWithMask, this->array("[0, 1, 2, null]"), - this->mask("[true, true, true, true]"), this->array("[10, 11, 12, 13]"), - this->array("[10, 11, 12, 13]")); - this->Assert(ReplaceWithMask, this->array("[0, 1, 2, null]"), - this->mask("[null, null, null, null]"), this->array("[]"), - this->array("[null, null, null, null]")); - this->Assert(ReplaceWithMask, this->array("[0, 1, 2, 3, 4, 5]"), - this->mask("[false, false, null, null, true, true]"), - this->array("[10, null]"), this->array("[0, 1, null, null, 10, null]")); - this->Assert(ReplaceWithMask, this->array("[null, null, null, null, null, null]"), - this->mask("[false, false, null, null, true, true]"), - this->array("[10, null]"), - this->array("[null, null, null, null, 10, null]")); - - this->Assert(ReplaceWithMask, this->array("[]"), this->mask("[]"), this->scalar("1"), - this->array("[]")); - this->Assert(ReplaceWithMask, this->array("[0, 1]"), this->mask("[true, true]"), - this->scalar("10"), this->array("[10, 10]")); - this->Assert(ReplaceWithMask, this->array("[0, 1]"), this->mask("[true, true]"), - this->scalar("null"), this->array("[null, null]")); - this->Assert(ReplaceWithMask, this->array("[0, 1, 2]"), - this->mask("[false, null, true]"), this->scalar("10"), - this->array("[0, null, 10]")); + if (std::is_same::value) { + this->Assert(ReplaceWithMask, this->array("[86400000]"), this->mask_scalar(false), + this->array("[]"), this->array("[86400000]")); + this->Assert(ReplaceWithMask, this->array("[86400000]"), this->mask_scalar(true), + this->array("[0]"), this->array("[0]")); + this->Assert(ReplaceWithMask, this->array("[86400000]"), this->mask_scalar(true), + this->array("[172800000, 0]"), this->array("[172800000]")); + this->Assert(ReplaceWithMask, this->array("[86400000]"), this->null_mask_scalar(), + this->array("[]"), this->array("[null]")); + + this->Assert(ReplaceWithMask, this->array("[0, 0]"), this->mask_scalar(false), + this->scalar("86400000"), this->array("[0, 0]")); + this->Assert(ReplaceWithMask, this->array("[0, 0]"), this->mask_scalar(true), + this->scalar("86400000"), this->array("[86400000, 86400000]")); + this->Assert(ReplaceWithMask, this->array("[0, 0]"), this->mask_scalar(true), + this->scalar("null"), this->array("[null, null]")); + + this->Assert(ReplaceWithMask, this->array("[]"), this->mask("[]"), this->array("[]"), + this->array("[]")); + this->Assert(ReplaceWithMask, this->array("[0, 86400000, 172800000, 259200000]"), + this->mask("[false, false, false, false]"), this->array("[]"), + this->array("[0, 86400000, 172800000, 259200000]")); + this->Assert(ReplaceWithMask, this->array("[0, 86400000, 172800000, 259200000]"), + this->mask("[true, true, true, true]"), + this->array("[864000000, 950400000, 1036800000, 1123200000]"), + this->array("[864000000, 950400000, 1036800000, 1123200000]")); + this->Assert(ReplaceWithMask, this->array("[0, 86400000, 172800000, 259200000]"), + this->mask("[null, null, null, null]"), this->array("[]"), + this->array("[null, null, null, null]")); + this->Assert(ReplaceWithMask, this->array("[0, 86400000, 172800000, null]"), + this->mask("[false, false, false, false]"), this->array("[]"), + this->array("[0, 86400000, 172800000, null]")); + this->Assert(ReplaceWithMask, this->array("[0, 86400000, 172800000, null]"), + this->mask("[true, true, true, true]"), + this->array("[864000000, 950400000, 1036800000, 1123200000]"), + this->array("[864000000, 950400000, 1036800000, 1123200000]")); + this->Assert(ReplaceWithMask, this->array("[0, 86400000, 172800000, null]"), + this->mask("[null, null, null, null]"), this->array("[]"), + this->array("[null, null, null, null]")); + this->Assert(ReplaceWithMask, + this->array("[0, 86400000, 172800000, 259200000, 345600000, 432000000]"), + this->mask("[false, false, null, null, true, true]"), + this->array("[864000000, null]"), + this->array("[0, 86400000, null, null, 864000000, null]")); + this->Assert(ReplaceWithMask, this->array("[null, null, null, null, null, null]"), + this->mask("[false, false, null, null, true, true]"), + this->array("[864000000, null]"), + this->array("[null, null, null, null, 864000000, null]")); + + this->Assert(ReplaceWithMask, this->array("[]"), this->mask("[]"), + this->scalar("86400000"), this->array("[]")); + this->Assert(ReplaceWithMask, this->array("[0, 86400000]"), + this->mask("[true, true]"), this->scalar("864000000"), + this->array("[864000000, 864000000]")); + this->Assert(ReplaceWithMask, this->array("[0, 86400000]"), + this->mask("[true, true]"), this->scalar("null"), + this->array("[null, null]")); + this->Assert(ReplaceWithMask, this->array("[0, 86400000, 172800000]"), + this->mask("[false, null, true]"), this->scalar("864000000"), + this->array("[0, null, 864000000]")); + } else { + this->Assert(ReplaceWithMask, this->array("[1]"), this->mask_scalar(false), + this->array("[]"), this->array("[1]")); + this->Assert(ReplaceWithMask, this->array("[1]"), this->mask_scalar(true), + this->array("[0]"), this->array("[0]")); + this->Assert(ReplaceWithMask, this->array("[1]"), this->mask_scalar(true), + this->array("[2, 0]"), this->array("[2]")); + this->Assert(ReplaceWithMask, this->array("[1]"), this->null_mask_scalar(), + this->array("[]"), this->array("[null]")); + + this->Assert(ReplaceWithMask, this->array("[0, 0]"), this->mask_scalar(false), + this->scalar("1"), this->array("[0, 0]")); + this->Assert(ReplaceWithMask, this->array("[0, 0]"), this->mask_scalar(true), + this->scalar("1"), this->array("[1, 1]")); + this->Assert(ReplaceWithMask, this->array("[0, 0]"), this->mask_scalar(true), + this->scalar("null"), this->array("[null, null]")); + + this->Assert(ReplaceWithMask, this->array("[]"), this->mask("[]"), this->array("[]"), + this->array("[]")); + this->Assert(ReplaceWithMask, this->array("[0, 1, 2, 3]"), + this->mask("[false, false, false, false]"), this->array("[]"), + this->array("[0, 1, 2, 3]")); + this->Assert(ReplaceWithMask, this->array("[0, 1, 2, 3]"), + this->mask("[true, true, true, true]"), this->array("[10, 11, 12, 13]"), + this->array("[10, 11, 12, 13]")); + this->Assert(ReplaceWithMask, this->array("[0, 1, 2, 3]"), + this->mask("[null, null, null, null]"), this->array("[]"), + this->array("[null, null, null, null]")); + this->Assert(ReplaceWithMask, this->array("[0, 1, 2, null]"), + this->mask("[false, false, false, false]"), this->array("[]"), + this->array("[0, 1, 2, null]")); + this->Assert(ReplaceWithMask, this->array("[0, 1, 2, null]"), + this->mask("[true, true, true, true]"), this->array("[10, 11, 12, 13]"), + this->array("[10, 11, 12, 13]")); + this->Assert(ReplaceWithMask, this->array("[0, 1, 2, null]"), + this->mask("[null, null, null, null]"), this->array("[]"), + this->array("[null, null, null, null]")); + this->Assert(ReplaceWithMask, this->array("[0, 1, 2, 3, 4, 5]"), + this->mask("[false, false, null, null, true, true]"), + this->array("[10, null]"), this->array("[0, 1, null, null, 10, null]")); + this->Assert(ReplaceWithMask, this->array("[null, null, null, null, null, null]"), + this->mask("[false, false, null, null, true, true]"), + this->array("[10, null]"), + this->array("[null, null, null, null, 10, null]")); + + this->Assert(ReplaceWithMask, this->array("[]"), this->mask("[]"), this->scalar("1"), + this->array("[]")); + this->Assert(ReplaceWithMask, this->array("[0, 1]"), this->mask("[true, true]"), + this->scalar("10"), this->array("[10, 10]")); + this->Assert(ReplaceWithMask, this->array("[0, 1]"), this->mask("[true, true]"), + this->scalar("null"), this->array("[null, null]")); + this->Assert(ReplaceWithMask, this->array("[0, 1, 2]"), + this->mask("[false, null, true]"), this->scalar("10"), + this->array("[0, null, 10]")); + } } TYPED_TEST(TestReplaceNumeric, ReplaceWithMaskForNullValuesAndMaskEnabled) { - this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), - this->mask("[false, true, false]"), this->array("[7]"), - this->array("[1, 7, 1]")); - this->Assert(ReplaceWithMask, this->array("[1, null, 1, 7]"), - this->mask("[false, true, false, true]"), this->array("[7, 20]"), - this->array("[1, 7, 1, 20]")); - this->Assert(ReplaceWithMask, this->array("[1, 2, 3, 4]"), - this->mask("[false, true, false, true]"), this->array("[null, null]"), - this->array("[1, null, 3, null]")); - this->Assert(ReplaceWithMask, this->array("[null, 2, 3, 4]"), - this->mask("[true, true, false, true]"), this->array("[1, null, null]"), - this->array("[1, null, 3, null]")); - this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), - this->mask("[false, true, false]"), this->scalar("null"), - this->array("[1, null, 1]")); - this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), - this->mask("[true, true, true]"), this->array("[7, 7, 7]"), - this->array("[7, 7, 7]")); - this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), - this->mask("[true, true, true]"), this->array("[null, null, null]"), - this->array("[null, null, null]")); - this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), - this->mask("[false, true, false]"), this->scalar("null"), - this->array("[1, null, 1]")); - this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), - this->mask("[true, true, true]"), this->scalar("null"), - this->array("[null, null, null]")); - this->Assert(ReplaceWithMask, this->array("[null, null]"), this->mask("[true, true]"), - this->array("[1, 1]"), this->array("[1, 1]")); + if (std::is_same::value) { + this->Assert(ReplaceWithMask, this->array("[86400000, null, 86400000]"), + this->mask("[false, true, false]"), this->array("[604800000]"), + this->array("[86400000, 604800000, 86400000]")); + this->Assert(ReplaceWithMask, this->array("[86400000, null, 86400000, 604800000]"), + this->mask("[false, true, false, true]"), + this->array("[604800000, 1728000000]"), + this->array("[86400000, 604800000, 86400000, 1728000000]")); + this->Assert(ReplaceWithMask, + this->array("[86400000, 172800000, 259200000, 345600000]"), + this->mask("[false, true, false, true]"), this->array("[null, null]"), + this->array("[86400000, null, 259200000, null]")); + this->Assert(ReplaceWithMask, this->array("[null, 172800000, 259200000, 345600000]"), + this->mask("[true, true, false, true]"), + this->array("[86400000, null, null]"), + this->array("[86400000, null, 259200000, null]")); + this->Assert(ReplaceWithMask, this->array("[86400000, null, 86400000]"), + this->mask("[false, true, false]"), this->scalar("null"), + this->array("[86400000, null, 86400000]")); + this->Assert(ReplaceWithMask, this->array("[86400000, null, 86400000]"), + this->mask("[true, true, true]"), + this->array("[604800000, 604800000, 604800000]"), + this->array("[604800000, 604800000, 604800000]")); + this->Assert(ReplaceWithMask, this->array("[86400000, null, 86400000]"), + this->mask("[true, true, true]"), this->array("[null, null, null]"), + this->array("[null, null, null]")); + this->Assert(ReplaceWithMask, this->array("[86400000, null, 86400000]"), + this->mask("[false, true, false]"), this->scalar("null"), + this->array("[86400000, null, 86400000]")); + this->Assert(ReplaceWithMask, this->array("[86400000, null, 86400000]"), + this->mask("[true, true, true]"), this->scalar("null"), + this->array("[null, null, null]")); + this->Assert(ReplaceWithMask, this->array("[null, null]"), this->mask("[true, true]"), + this->array("[86400000, 86400000]"), + this->array("[86400000, 86400000]")); + } else { + this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), + this->mask("[false, true, false]"), this->array("[7]"), + this->array("[1, 7, 1]")); + this->Assert(ReplaceWithMask, this->array("[1, null, 1, 7]"), + this->mask("[false, true, false, true]"), this->array("[7, 20]"), + this->array("[1, 7, 1, 20]")); + this->Assert(ReplaceWithMask, this->array("[1, 2, 3, 4]"), + this->mask("[false, true, false, true]"), this->array("[null, null]"), + this->array("[1, null, 3, null]")); + this->Assert(ReplaceWithMask, this->array("[null, 2, 3, 4]"), + this->mask("[true, true, false, true]"), this->array("[1, null, null]"), + this->array("[1, null, 3, null]")); + this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), + this->mask("[false, true, false]"), this->scalar("null"), + this->array("[1, null, 1]")); + this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), + this->mask("[true, true, true]"), this->array("[7, 7, 7]"), + this->array("[7, 7, 7]")); + this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), + this->mask("[true, true, true]"), this->array("[null, null, null]"), + this->array("[null, null, null]")); + this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), + this->mask("[false, true, false]"), this->scalar("null"), + this->array("[1, null, 1]")); + this->Assert(ReplaceWithMask, this->array("[1, null, 1]"), + this->mask("[true, true, true]"), this->scalar("null"), + this->array("[null, null, null]")); + this->Assert(ReplaceWithMask, this->array("[null, null]"), this->mask("[true, true]"), + this->array("[1, 1]"), this->array("[1, 1]")); + } } TYPED_TEST(TestReplaceNumeric, ReplaceWithMaskRandom) { @@ -879,37 +978,92 @@ TYPED_TEST(TestFillNullNumeric, FillNullValuesForward) { this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, null]"), this->array("[null, null, null, null]")); - this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"), - this->array("[null, null, null, 4]")); - this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"), - this->array("[null, 4, 4]")); - this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"), - this->array("[null, null, 4, 4]")); - this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"), - this->array("[null, null, null, 4, 4]")); - this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4,null, 5, null]"), - this->array("[null, null, 4, 4, 5, 5]")); - - this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"), - this->array("[1,4,4]")); - this->AssertFillNullArray(FillNullForward, - this->array("[1, 4, null, null, null, null]"), - this->array("[1, 4 ,4, 4, 4, 4]")); - this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"), - this->array("[1, 4 ,4, 5, 5, 5]")); - this->AssertFillNullArray(FillNullForward, - this->array("[1, 4, null, 5, null, null, 6]"), - this->array("[1, 4 ,4, 5, 5, 5, 6]")); - this->AssertFillNullArray(FillNullForward, - this->array("[1, 4, null, 5, null, null, 5]"), - this->array("[1, 4 ,4, 5, 5, 5, 5]")); - this->AssertFillNullArray(FillNullForward, - this->array("[1, 4, null, 5, null, 6, null]"), - this->array("[1, 4 ,4, 5, 5, 6, 6]")); - this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"), - this->array("[1, 4 ,4, 5, 5, 6, 7]")); - this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"), - this->array("[1, 4 ,4, 5, 5, 6, 7]")); + + if (std::is_same::value) { + this->AssertFillNullArray(FillNullForward, + this->array("[null, null, null, 345600000]"), + this->array("[null, null, null, 345600000]")); + this->AssertFillNullArray(FillNullForward, this->array("[null, 345600000, null]"), + this->array("[null, 345600000, 345600000]")); + this->AssertFillNullArray(FillNullForward, + this->array("[null, null, 345600000, null]"), + this->array("[null, null, 345600000, 345600000]")); + this->AssertFillNullArray(FillNullForward, + this->array("[null, null, null, 345600000, null]"), + this->array("[null, null, null, 345600000, 345600000]")); + this->AssertFillNullArray( + FillNullForward, this->array("[null, null, 345600000, null, 432000000, null]"), + this->array("[null, null, 345600000, 345600000, 432000000, 432000000]")); + + this->AssertFillNullArray(FillNullForward, this->array("[86400000, 345600000, null]"), + this->array("[86400000, 345600000, 345600000]")); + this->AssertFillNullArray( + FillNullForward, this->array("[86400000, 345600000, null, null, null, null]"), + this->array("[86400000, 345600000 ,345600000, 345600000, 345600000, 345600000]")); + this->AssertFillNullArray( + FillNullForward, + this->array("[86400000, 345600000, null, 432000000, null, null]"), + this->array("[86400000, 345600000 ,345600000, 432000000, 432000000, 432000000]")); + this->AssertFillNullArray( + FillNullForward, + this->array("[86400000, 345600000, null, 432000000, null, null, 518400000]"), + this->array("[86400000, 345600000 ,345600000, 432000000, 432000000, 432000000, " + "518400000]")); + this->AssertFillNullArray( + FillNullForward, + this->array("[86400000, 345600000, null, 432000000, null, null, 432000000]"), + this->array("[86400000, 345600000 ,345600000, 432000000, 432000000, 432000000, " + "432000000]")); + this->AssertFillNullArray( + FillNullForward, + this->array("[86400000, 345600000, null, 432000000, null, 518400000, null]"), + this->array("[86400000, 345600000 ,345600000, 432000000, 432000000, 518400000, " + "518400000]")); + this->AssertFillNullArray( + FillNullForward, + this->array("[86400000, 345600000, null, 432000000, null, 518400000, 604800000]"), + this->array("[86400000, 345600000 ,345600000, 432000000, 432000000, 518400000, " + "604800000]")); + this->AssertFillNullArray( + FillNullForward, + this->array("[86400000, 345600000 ,345600000, 432000000, 432000000, 518400000, " + "604800000]"), + this->array("[86400000, 345600000 ,345600000, 432000000, 432000000, 518400000, " + "604800000]")); + } else { + this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4]"), + this->array("[null, null, null, 4]")); + this->AssertFillNullArray(FillNullForward, this->array("[null, 4, null]"), + this->array("[null, 4, 4]")); + this->AssertFillNullArray(FillNullForward, this->array("[null, null, 4, null]"), + this->array("[null, null, 4, 4]")); + this->AssertFillNullArray(FillNullForward, this->array("[null, null, null, 4, null]"), + this->array("[null, null, null, 4, 4]")); + this->AssertFillNullArray(FillNullForward, + this->array("[null, null, 4,null, 5, null]"), + this->array("[null, null, 4, 4, 5, 5]")); + + this->AssertFillNullArray(FillNullForward, this->array("[1,4,null]"), + this->array("[1,4,4]")); + this->AssertFillNullArray(FillNullForward, + this->array("[1, 4, null, null, null, null]"), + this->array("[1, 4 ,4, 4, 4, 4]")); + this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, null]"), + this->array("[1, 4 ,4, 5, 5, 5]")); + this->AssertFillNullArray(FillNullForward, + this->array("[1, 4, null, 5, null, null, 6]"), + this->array("[1, 4 ,4, 5, 5, 5, 6]")); + this->AssertFillNullArray(FillNullForward, + this->array("[1, 4, null, 5, null, null, 5]"), + this->array("[1, 4 ,4, 5, 5, 5, 5]")); + this->AssertFillNullArray(FillNullForward, + this->array("[1, 4, null, 5, null, 6, null]"), + this->array("[1, 4 ,4, 5, 5, 6, 6]")); + this->AssertFillNullArray(FillNullForward, this->array("[1, 4, null, 5, null, 6, 7]"), + this->array("[1, 4 ,4, 5, 5, 6, 7]")); + this->AssertFillNullArray(FillNullForward, this->array("[1, 4 ,4, 5, 5, 6, 7]"), + this->array("[1, 4 ,4, 5, 5, 6, 7]")); + } } TYPED_TEST(TestFillNullDecimal, FillNullValuesForward) { @@ -1010,40 +1164,104 @@ TYPED_TEST(TestFillNullNumeric, FillNullValuesBackward) { this->AssertFillNullArray(FillNullBackward, this->array("[]"), this->array("[]")); this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, null]"), this->array("[null, null, null, null]")); - this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null, null, null]"), - this->array("[4, 4,null, null, null]")); - this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"), - this->array("[4, 4, 4, 4]")); - this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"), - this->array("[4, 4, null]")); - this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"), - this->array("[4, 4, 4, null]")); - this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4, null]"), - this->array("[4, 4, 4, 4, null]")); - this->AssertFillNullArray(FillNullBackward, - this->array("[null, null, 4,null, 5, null]"), - this->array("[4, 4, 4, 5, 5, null]")); - this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"), - this->array("[1, 4, null]")); - this->AssertFillNullArray(FillNullBackward, - this->array("[1, 4, null, null, null, null]"), - this->array("[1, 4 ,null, null, null, null]")); - this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, null]"), - this->array("[1, 4 , 5, 5, null, null]")); - this->AssertFillNullArray(FillNullBackward, - this->array("[1, 4, null, 5, null, null, 6]"), - this->array("[1, 4 ,5, 5, 6, 6, 6]")); - this->AssertFillNullArray(FillNullBackward, - this->array("[1, 4, null, 5, null, null, 5]"), - this->array("[1, 4 ,5 , 5, 5, 5, 5]")); - this->AssertFillNullArray(FillNullBackward, - this->array("[1, 4, null, 5, null, 6, null]"), - this->array("[1, 4 ,5 , 5, 6, 6, null]")); - this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null, 5, null, 6, 7]"), - this->array("[1, 4 ,5, 5, 6, 6, 7]")); - this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"), - this->array("[1, 4 ,5, 5, 6, 6, 7]")); + if (std::is_same::value) { + this->AssertFillNullArray(FillNullBackward, + this->array("[null, 345600000, null, null, null]"), + this->array("[345600000, 345600000,null, null, null]")); + this->AssertFillNullArray( + FillNullBackward, this->array("[null, null, null, 345600000]"), + this->array("[345600000, 345600000, 345600000, 345600000]")); + this->AssertFillNullArray(FillNullBackward, this->array("[null, 345600000, null]"), + this->array("[345600000, 345600000, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[null, null, 345600000, null]"), + this->array("[345600000, 345600000, 345600000, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[null, null, null, 345600000, null]"), + this->array("[345600000, 345600000, 345600000, 345600000, " + "null]")); + this->AssertFillNullArray( + FillNullBackward, this->array("[null, null, 345600000,null, 432000000, null]"), + this->array("[345600000, 345600000, 345600000, 432000000, " + "432000000, null]")); + + this->AssertFillNullArray(FillNullBackward, + this->array("[86400000, 345600000, null]"), + this->array("[86400000, 345600000, null]")); + this->AssertFillNullArray( + FillNullBackward, this->array("[86400000, 345600000, null, null, null, null]"), + this->array("[86400000, 345600000 ,null, null, null, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[86400000, 345600000, null, 432000000, null, " + "null]"), + this->array("[86400000, 345600000 , 432000000, 432000000, " + "null, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[86400000, 345600000, null, 432000000, null," + "null, 518400000]"), + this->array("[86400000, 345600000 ,432000000, 432000000, " + "518400000, 518400000, 518400000]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[86400000, 345600000, null, 432000000, null, " + "null, 432000000]"), + this->array("[86400000, 345600000 ,432000000 , 432000000, " + "432000000, 432000000, 432000000]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[86400000, 345600000, null, 432000000, null, " + "518400000, null]"), + this->array("[86400000, 345600000 ,432000000 , 432000000, " + "518400000, 518400000, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[86400000, 345600000, null, 432000000, null, " + "518400000, 604800000]"), + this->array("[86400000, 345600000 ,432000000, 432000000, " + "518400000, 518400000, 604800000]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[86400000, 345600000, 432000000, 432000000, " + "518400000, 518400000, 604800000]"), + this->array("[86400000, 345600000 ,432000000, 432000000, " + "518400000, 518400000, 604800000]")); + } else { + this->AssertFillNullArray(FillNullBackward, + this->array("[null, 4, null, null, null]"), + this->array("[4, 4,null, null, null]")); + this->AssertFillNullArray(FillNullBackward, this->array("[null, null, null, 4]"), + this->array("[4, 4, 4, 4]")); + this->AssertFillNullArray(FillNullBackward, this->array("[null, 4, null]"), + this->array("[4, 4, null]")); + this->AssertFillNullArray(FillNullBackward, this->array("[null, null, 4, null]"), + this->array("[4, 4, 4, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[null, null, null, 4, null]"), + this->array("[4, 4, 4, 4, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[null, null, 4,null, 5, null]"), + this->array("[4, 4, 4, 5, 5, null]")); + + this->AssertFillNullArray(FillNullBackward, this->array("[1, 4, null]"), + this->array("[1, 4, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[1, 4, null, null, null, null]"), + this->array("[1, 4 ,null, null, null, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[1, 4, null, 5, null, null]"), + this->array("[1, 4 , 5, 5, null, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[1, 4, null, 5, null, null, 6]"), + this->array("[1, 4 ,5, 5, 6, 6, 6]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[1, 4, null, 5, null, null, 5]"), + this->array("[1, 4 ,5 , 5, 5, 5, 5]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[1, 4, null, 5, null, 6, null]"), + this->array("[1, 4 ,5 , 5, 6, 6, null]")); + this->AssertFillNullArray(FillNullBackward, + this->array("[1, 4, null, 5, null, 6, 7]"), + this->array("[1, 4 ,5, 5, 6, 6, 7]")); + this->AssertFillNullArray(FillNullBackward, this->array("[1, 4 ,5, 5, 6, 6, 7]"), + this->array("[1, 4 ,5, 5, 6, 6, 7]")); + } } TYPED_TEST(TestFillNullDecimal, FillNullValuesBackward) { @@ -1190,15 +1408,33 @@ TYPED_TEST(TestFillNullNumeric, FillNullBackwardLargeInput) { } TYPED_TEST(TestFillNullNumeric, FillNullForwardSliced) { - auto first_input_array = - this->array("[1, 4, null, null, 7, null, null, 8, 9, 5, null]"); - auto expected_slice_length_2 = - this->array("[1, 4, null, null, 7, 7, null, 8, 9, 5, null]"); - auto expected_slice_length_3 = this->array("[1, 4, 4, null, 7, 7, null, 8, 9, 5, 5]"); - auto expected_slice_length_4 = this->array("[1, 4, 4, 4, 7, 7, 7, 8, 9, 5, 5]"); - this->AssertFillNullArraySlices(FillNullForward, first_input_array, - expected_slice_length_2, expected_slice_length_3, - expected_slice_length_4); + if (std::is_same::value) { + auto first_input_array = this->array( + "[86400000, 345600000, null, null, 604800000, null, null, 691200000, " + "777600000, 432000000, null]"); + auto expected_slice_length_2 = this->array( + "[86400000, 345600000, null, null, 604800000, 604800000, null, " + "691200000, 777600000, 432000000, null]"); + auto expected_slice_length_3 = this->array( + "[86400000, 345600000, 345600000, null, 604800000, 604800000, null, " + "691200000, 777600000, 432000000, 432000000]"); + auto expected_slice_length_4 = this->array( + "[86400000, 345600000, 345600000, 345600000, 604800000, 604800000, " + "604800000, 691200000, 777600000, 432000000, 432000000]"); + this->AssertFillNullArraySlices(FillNullForward, first_input_array, + expected_slice_length_2, expected_slice_length_3, + expected_slice_length_4); + } else { + auto first_input_array = + this->array("[1, 4, null, null, 7, null, null, 8, 9, 5, null]"); + auto expected_slice_length_2 = + this->array("[1, 4, null, null, 7, 7, null, 8, 9, 5, null]"); + auto expected_slice_length_3 = this->array("[1, 4, 4, null, 7, 7, null, 8, 9, 5, 5]"); + auto expected_slice_length_4 = this->array("[1, 4, 4, 4, 7, 7, 7, 8, 9, 5, 5]"); + this->AssertFillNullArraySlices(FillNullForward, first_input_array, + expected_slice_length_2, expected_slice_length_3, + expected_slice_length_4); + } } TYPED_TEST(TestFillNullBinary, FillNullForwardSliced) { @@ -1217,18 +1453,37 @@ TYPED_TEST(TestFillNullBinary, FillNullForwardSliced) { } TYPED_TEST(TestFillNullNumeric, FillNullBackwardSliced) { - auto first_input_array = - this->array("[1, 4, null, null, 7, null, null, 8, 9, 5, null]"); - auto expected_slice_length_2 = - this->array("[1, 4, null, null, 7, null, 8, 8, 9, 5, null]"); - auto expected_slice_length_3 = - this->array("[1, 4, null, 7, 7, null, 8, 8, 9, 5, null]"); - auto expected_slice_length_4 = - this->array("[1, 4, null, null, 7, 8, 8, 8, 9, 5, null]"); - - this->AssertFillNullArraySlices(FillNullBackward, first_input_array, - expected_slice_length_2, expected_slice_length_3, - expected_slice_length_4); + if (std::is_same::value) { + auto first_input_array = this->array( + "[86400000, 345600000, null, null, 604800000, null, null, 691200000, " + "777600000, 432000000, null]"); + auto expected_slice_length_2 = this->array( + "[86400000, 345600000, null, null, 604800000, null, 691200000, " + "691200000, 777600000, 432000000, null]"); + auto expected_slice_length_3 = this->array( + "[86400000, 345600000, null, 604800000, 604800000, null, 691200000, " + "691200000, 777600000, 432000000, null]"); + auto expected_slice_length_4 = this->array( + "[86400000, 345600000, null, null, 604800000, 691200000, 691200000, " + "691200000, 777600000, 432000000, null]"); + + this->AssertFillNullArraySlices(FillNullBackward, first_input_array, + expected_slice_length_2, expected_slice_length_3, + expected_slice_length_4); + } else { + auto first_input_array = + this->array("[1, 4, null, null, 7, null, null, 8, 9, 5, null]"); + auto expected_slice_length_2 = + this->array("[1, 4, null, null, 7, null, 8, 8, 9, 5, null]"); + auto expected_slice_length_3 = + this->array("[1, 4, null, 7, 7, null, 8, 8, 9, 5, null]"); + auto expected_slice_length_4 = + this->array("[1, 4, null, null, 7, 8, 8, 8, 9, 5, null]"); + + this->AssertFillNullArraySlices(FillNullBackward, first_input_array, + expected_slice_length_2, expected_slice_length_3, + expected_slice_length_4); + } } TYPED_TEST(TestFillNullBinary, FillNullBackwardSliced) { @@ -1253,61 +1508,149 @@ TYPED_TEST(TestFillNullNumeric, FillNullForwardChunkedArray) { {"[null, null, null]", "[null, null, null]", "[null]", "[null, null]"}), this->chunked_array( {"[null, null, null]", "[null, null, null]", "[null]", "[null, null]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array( - {"[7, null, null]", "[null, null, null]", "[null]", "[null, null]"}), - this->chunked_array({"[7, 7, 7]", "[7, 7, 7]", "[7]", "[7, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array( - {"[7, null, null]", "[null, null, null]", "[]", "[null, null]"}), - this->chunked_array({"[7, 7, 7]", "[7, 7, 7]", "[]", "[7, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array( - {"[null, null, null]", "[null, null, null]", "[null]", "[null, 7]"}), - this->chunked_array( - {"[null, null, null]", "[null, null, null]", "[null]", "[null, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array( - {"[null, null, null]", "[8, null, null]", "[null]", "[null, 7]"}), - this->chunked_array({"[null, null, null]", "[8, 8, 8]", "[8]", "[8, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array( - {"[1, 2, 3, null, null, 4]", "[5, null, null]", "[null, 7, null]"}), - this->chunked_array({"[1, 2, 3, 3, 3, 4]", "[5, 5, 5]", "[5, 7, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array( - {"[1, 2, 3, null, null, null]", "[null, null, null]", "[null, 7, null]"}), - this->chunked_array({"[1, 2, 3, 3, 3, 3]", "[3, 3, 3]", "[3, 7, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array({"[1, 2, null]", "[null, null, null]", "[null]", "[null, 7]"}), - this->chunked_array({"[1, 2, 2]", "[2, 2, 2]", "[2]", "[2, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array({"[null, 2, null]", "[null, null, 3]", "[null]", "[null, 7]"}), - this->chunked_array({"[null, 2, 2]", "[2, 2, 3]", "[3]", "[3, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array({"[null, 2, null]", "[4, null, 3]", "[null]", "[null, 7]"}), - this->chunked_array({"[null, 2, 2]", "[4, 4, 3]", "[3]", "[3, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array({"[null, null, null]", "[null, null, null]", "[5]", "[5, 7]"}), - this->chunked_array({"[null, null, null]", "[null, null, null]", "[5]", "[5, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array({"[null, null, 1]", "[null, null, null]", "[5]", "[5, 7]"}), - this->chunked_array({"[null, null, 1]", "[1, 1, 1]", "[5]", "[5, 7]"})); - this->AssertFillNullChunkedArray( - FillNullForward, - this->chunked_array({"[2, 4, 1]", "[null, null, null]", "[5]", "[5, 7]"}), - this->chunked_array({"[2, 4, 1]", "[1, 1, 1]", "[5]", "[5, 7]"})); + + if (std::is_same::value) { + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[604800000, null, null]", "[null, null, null]", "[null]", "[null, null]"}), + this->chunked_array({"[604800000, 604800000, 604800000]", + "[604800000, 604800000, 604800000]", "[604800000]", + "[604800000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[604800000, null, null]", "[null, null, null]", "[]", "[null, null]"}), + this->chunked_array({"[604800000, 604800000, 604800000]", + "[604800000, 604800000, 604800000]", "[]", + "[604800000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[null, null, null]", "[null, null, null]", "[null]", "[null, 604800000]"}), + this->chunked_array( + {"[null, null, null]", "[null, null, null]", "[null]", "[null, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[null, null, null]", "[691200000, null, null]", "[null]", + "[null, 604800000]"}), + this->chunked_array({"[null, null, null]", "[691200000, 691200000, 691200000]", + "[691200000]", "[691200000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[86400000, 172800000, 259200000, null, null, 345600000]", + "[432000000, null, null]", "[null, 604800000, null]"}), + this->chunked_array( + {"[86400000, 172800000, 259200000, 259200000, 259200000, 345600000]", + "[432000000, 432000000, 432000000]", "[432000000, 604800000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[86400000, 172800000, 259200000, null, null, null]", + "[null, null, null]", "[null, 604800000, null]"}), + this->chunked_array( + {"[86400000, 172800000, 259200000, 259200000, 259200000, 259200000]", + "[259200000, 259200000, 259200000]", "[259200000, 604800000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[86400000, 172800000, null]", "[null, null, null]", + "[null]", "[null, 604800000]"}), + this->chunked_array({"[86400000, 172800000, 172800000]", + "[172800000, 172800000, 172800000]", "[172800000]", + "[172800000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[null, 172800000, null]", "[null, null, 259200000]", + "[null]", "[null, 604800000]"}), + this->chunked_array({"[null, 172800000, 172800000]", + "[172800000, 172800000, 259200000]", "[259200000]", + "[259200000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[null, 172800000, null]", "[345600000, null, 259200000]", + "[null]", "[null, 604800000]"}), + this->chunked_array({"[null, 172800000, 172800000]", + "[345600000, 345600000, 259200000]", "[259200000]", + "[259200000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[null, null, null]", "[null, null, null]", "[432000000]", + "[432000000, 604800000]"}), + this->chunked_array({"[null, null, null]", "[null, null, null]", "[432000000]", + "[432000000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[null, null, 86400000]", "[null, null, null]", + "[432000000]", "[432000000, 604800000]"}), + this->chunked_array({"[null, null, 86400000]", "[86400000, 86400000, 86400000]", + "[432000000]", "[432000000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[172800000, 345600000, 86400000]", "[null, null, null]", + "[432000000]", "[432000000, 604800000]"}), + this->chunked_array({"[172800000, 345600000, 86400000]", + "[86400000, 86400000, 86400000]", "[432000000]", + "[432000000, 604800000]"})); + } else { + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[7, null, null]", "[null, null, null]", "[null]", "[null, null]"}), + this->chunked_array({"[7, 7, 7]", "[7, 7, 7]", "[7]", "[7, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[7, null, null]", "[null, null, null]", "[]", "[null, null]"}), + this->chunked_array({"[7, 7, 7]", "[7, 7, 7]", "[]", "[7, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[null, null, null]", "[null, null, null]", "[null]", "[null, 7]"}), + this->chunked_array( + {"[null, null, null]", "[null, null, null]", "[null]", "[null, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[null, null, null]", "[8, null, null]", "[null]", "[null, 7]"}), + this->chunked_array({"[null, null, null]", "[8, 8, 8]", "[8]", "[8, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[1, 2, 3, null, null, 4]", "[5, null, null]", "[null, 7, null]"}), + this->chunked_array({"[1, 2, 3, 3, 3, 4]", "[5, 5, 5]", "[5, 7, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[1, 2, 3, null, null, null]", "[null, null, null]", "[null, 7, null]"}), + this->chunked_array({"[1, 2, 3, 3, 3, 3]", "[3, 3, 3]", "[3, 7, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[1, 2, null]", "[null, null, null]", "[null]", "[null, 7]"}), + this->chunked_array({"[1, 2, 2]", "[2, 2, 2]", "[2]", "[2, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[null, 2, null]", "[null, null, 3]", "[null]", "[null, 7]"}), + this->chunked_array({"[null, 2, 2]", "[2, 2, 3]", "[3]", "[3, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[null, 2, null]", "[4, null, 3]", "[null]", "[null, 7]"}), + this->chunked_array({"[null, 2, 2]", "[4, 4, 3]", "[3]", "[3, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array( + {"[null, null, null]", "[null, null, null]", "[5]", "[5, 7]"}), + this->chunked_array( + {"[null, null, null]", "[null, null, null]", "[5]", "[5, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[null, null, 1]", "[null, null, null]", "[5]", "[5, 7]"}), + this->chunked_array({"[null, null, 1]", "[1, 1, 1]", "[5]", "[5, 7]"})); + this->AssertFillNullChunkedArray( + FillNullForward, + this->chunked_array({"[2, 4, 1]", "[null, null, null]", "[5]", "[5, 7]"}), + this->chunked_array({"[2, 4, 1]", "[1, 1, 1]", "[5]", "[5, 7]"})); + } } TYPED_TEST(TestFillNullBinary, FillNullForwardChunkedArray) { @@ -1395,56 +1738,140 @@ TYPED_TEST(TestFillNullNumeric, FillBackwardChunkedArray) { {"[null, null, null]", "[null, null, null]", "[null]", "[null, null]"}), this->chunked_array( {"[null, null, null]", "[null, null, null]", "[null]", "[null, null]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array( - {"[7, null, null]", "[null, null, null]", "[null]", "[null, null]"}), - this->chunked_array( - {"[7, null, null]", "[null, null, null]", "[null]", "[null, null]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array( - {"[null, null, null]", "[null, null, null]", "[null]", "[null, 7]"}), - this->chunked_array({"[7, 7, 7]", "[7, 7, 7]", "[7]", "[7, 7]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array( - {"[null, null, null]", "[8, null, null]", "[null]", "[null, 7]"}), - this->chunked_array({"[8, 8, 8]", "[8, 7, 7]", "[7]", "[7, 7]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array( - {"[1, 2, 3, null, null, 4]", "[5, null, null]", "[null, 7, null]"}), - this->chunked_array({"[1, 2, 3, 4, 4, 4]", "[5, 7, 7]", "[7, 7, null]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array( - {"[1, 2, 3, null, null, null]", "[null, null, null]", "[null, 7, null]"}), - this->chunked_array({"[1, 2, 3, 7, 7, 7]", "[7, 7, 7]", "[7, 7, null]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array({"[1, 2, null]", "[null, null, null]", "[null]", "[null, 7]"}), - this->chunked_array({"[1, 2, 7]", "[7, 7, 7]", "[7]", "[7, 7]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array({"[null, 2, null]", "[null, null, 3]", "[null]", "[null, 7]"}), - this->chunked_array({"[2, 2, 3]", "[3, 3, 3]", "[7]", "[7, 7]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array({"[null, 2, null]", "[4, null, 3]", "[null]", "[null, 7]"}), - this->chunked_array({"[2, 2, 4]", "[4, 3, 3]", "[7]", "[7, 7]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array({"[null, null, null]", "[null, null, null]", "[5]", "[5, 7]"}), - this->chunked_array({"[5, 5, 5]", "[5, 5, 5]", "[5]", "[5, 7]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array({"[null, null, 1]", "[null, null, null]", "[5]", "[5, 7]"}), - this->chunked_array({"[1, 1, 1]", "[5, 5, 5]", "[5]", "[5, 7]"})); - this->AssertFillNullChunkedArray( - FillNullBackward, - this->chunked_array({"[null, null, 1]", "[null, null, null]", "[9, 0]", "[5, 7]"}), - this->chunked_array({"[1, 1, 1]", "[9, 9, 9]", "[9, 0]", "[5, 7]"})); + + if (std::is_same::value) { + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[604800000, null, null]", "[null, null, null]", "[null]", "[null, null]"}), + this->chunked_array( + {"[604800000, null, null]", "[null, null, null]", "[null]", "[null, null]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[null, null, null]", "[null, null, null]", "[null]", "[null, 604800000]"}), + this->chunked_array({"[604800000, 604800000, 604800000]", + "[604800000, 604800000, 604800000]", "[604800000]", + "[604800000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[null, null, null]", "[691200000, null, null]", "[null]", + "[null, 604800000]"}), + this->chunked_array({"[691200000, 691200000, 691200000]", + "[691200000, 604800000, 604800000]", "[604800000]", + "[604800000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[86400000, 172800000, 259200000, null, null, 345600000]", + "[432000000, null, null]", "[null, 604800000, null]"}), + this->chunked_array( + {"[86400000, 172800000, 259200000, 345600000, 345600000, 345600000]", + "[432000000, 604800000, 604800000]", "[604800000, 604800000, null]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[86400000, 172800000, 259200000, null, null, null]", + "[null, null, null]", "[null, 604800000, null]"}), + this->chunked_array( + {"[86400000, 172800000, 259200000, 604800000, 604800000, 604800000]", + "[604800000, 604800000, 604800000]", "[604800000, 604800000, null]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[86400000, 172800000, null]", "[null, null, null]", + "[null]", "[null, 604800000]"}), + this->chunked_array({"[86400000, 172800000, 604800000]", + "[604800000, 604800000, 604800000]", "[604800000]", + "[604800000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[null, 172800000, null]", "[null, null, 259200000]", + "[null]", "[null, 604800000]"}), + this->chunked_array({"[172800000, 172800000, 259200000]", + "[259200000, 259200000, 259200000]", "[604800000]", + "[604800000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[null, 172800000, null]", "[345600000, null, 259200000]", + "[null]", "[null, 604800000]"}), + this->chunked_array({"[172800000, 172800000, 345600000]", + "[345600000, 259200000, 259200000]", "[604800000]", + "[604800000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[null, null, null]", "[null, null, null]", "[432000000]", + "[432000000, 604800000]"}), + this->chunked_array({"[432000000, 432000000, 432000000]", + "[432000000, 432000000, 432000000]", "[432000000]", + "[432000000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[null, null, 86400000]", "[null, null, null]", + "[432000000]", "[432000000, 604800000]"}), + this->chunked_array({"[86400000, 86400000, 86400000]", + "[432000000, 432000000, 432000000]", "[432000000]", + "[432000000, 604800000]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[null, null, 86400000]", "[null, null, null]", + "[777600000, 0]", "[432000000, 604800000]"}), + this->chunked_array({"[86400000, 86400000, 86400000]", + "[777600000, 777600000, 777600000]", "[777600000, 0]", + "[432000000, 604800000]"})); + } else { + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[7, null, null]", "[null, null, null]", "[null]", "[null, null]"}), + this->chunked_array( + {"[7, null, null]", "[null, null, null]", "[null]", "[null, null]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[null, null, null]", "[null, null, null]", "[null]", "[null, 7]"}), + this->chunked_array({"[7, 7, 7]", "[7, 7, 7]", "[7]", "[7, 7]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[null, null, null]", "[8, null, null]", "[null]", "[null, 7]"}), + this->chunked_array({"[8, 8, 8]", "[8, 7, 7]", "[7]", "[7, 7]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[1, 2, 3, null, null, 4]", "[5, null, null]", "[null, 7, null]"}), + this->chunked_array({"[1, 2, 3, 4, 4, 4]", "[5, 7, 7]", "[7, 7, null]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[1, 2, 3, null, null, null]", "[null, null, null]", "[null, 7, null]"}), + this->chunked_array({"[1, 2, 3, 7, 7, 7]", "[7, 7, 7]", "[7, 7, null]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[1, 2, null]", "[null, null, null]", "[null]", "[null, 7]"}), + this->chunked_array({"[1, 2, 7]", "[7, 7, 7]", "[7]", "[7, 7]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[null, 2, null]", "[null, null, 3]", "[null]", "[null, 7]"}), + this->chunked_array({"[2, 2, 3]", "[3, 3, 3]", "[7]", "[7, 7]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[null, 2, null]", "[4, null, 3]", "[null]", "[null, 7]"}), + this->chunked_array({"[2, 2, 4]", "[4, 3, 3]", "[7]", "[7, 7]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[null, null, null]", "[null, null, null]", "[5]", "[5, 7]"}), + this->chunked_array({"[5, 5, 5]", "[5, 5, 5]", "[5]", "[5, 7]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array({"[null, null, 1]", "[null, null, null]", "[5]", "[5, 7]"}), + this->chunked_array({"[1, 1, 1]", "[5, 5, 5]", "[5]", "[5, 7]"})); + this->AssertFillNullChunkedArray( + FillNullBackward, + this->chunked_array( + {"[null, null, 1]", "[null, null, null]", "[9, 0]", "[5, 7]"}), + this->chunked_array({"[1, 1, 1]", "[9, 9, 9]", "[9, 0]", "[5, 7]"})); + } } TYPED_TEST(TestFillNullBinary, FillBackwardChunkedArray) { From 27411f274235bbf95ffd3798c31e3124f96cb6bd Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Thu, 27 Jan 2022 22:18:10 -0500 Subject: [PATCH 14/17] Missing changes to constant from last commit --- .../compute/kernels/scalar_if_else_test.cc | 187 +++++++++++++----- 1 file changed, 136 insertions(+), 51 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc b/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc index 81d1341990b..abae98caf6d 100644 --- a/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_if_else_test.cc @@ -1143,7 +1143,7 @@ TYPED_TEST(TestCaseWhenNumeric, FixedSize) { auto scalar1 = ScalarFromJSON(type, "1"); auto scalar2 = ScalarFromJSON(type, "2"); auto values1 = ArrayFromJSON(type, "[3, null, 5, 6]"); - auto values2 = ArrayFromJSON(type, "[604800000, 8, null, 10]"); + auto values2 = ArrayFromJSON(type, "[7, 8, null, 10]"); CheckScalar("case_when", {MakeStruct({}), values1}, values1); CheckScalar("case_when", {MakeStruct({}), values_null}, values_null); @@ -1234,15 +1234,32 @@ TYPED_TEST(TestCaseWhenNumeric, ListOfType) { auto cond1 = ArrayFromJSON(boolean(), "[true, true, null, null]"); auto cond2 = ArrayFromJSON(boolean(), "[true, false, true, null]"); auto values_null = ArrayFromJSON(type, "[null, null, null, null]"); - auto values1 = ArrayFromJSON(type, R"([[1, 2], null, [3, 4, 5], [6, null]])"); - auto values2 = ArrayFromJSON(type, R"([[8, 9, 10], [11], null, [12]])"); - CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2}, - ArrayFromJSON(type, R"([[1, 2], null, null, null])")); - CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2, values1}, - ArrayFromJSON(type, R"([[1, 2], null, null, [6, null]])")); - CheckScalar("case_when", {MakeStruct({cond1, cond2}), values_null, values2, values1}, - ArrayFromJSON(type, R"([null, null, null, [6, null]])")); + if (std::is_same::value) { + auto values1 = ArrayFromJSON( + type, + R"([[86400000, 172800000], null, [259200000, 345600000, 432000000], [518400000, null]])"); + auto values2 = ArrayFromJSON( + type, R"([[691200000, 777600000, 864000000], [950400000], null, [1036800000]])"); + + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2}, + ArrayFromJSON(type, R"([[86400000, 172800000], null, null, null])")); + CheckScalar( + "case_when", {MakeStruct({cond1, cond2}), values1, values2, values1}, + ArrayFromJSON(type, R"([[86400000, 172800000], null, null, [518400000, null]])")); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values_null, values2, values1}, + ArrayFromJSON(type, R"([null, null, null, [518400000, null]])")); + } else { + auto values1 = ArrayFromJSON(type, R"([[1, 2], null, [3, 4, 5], [6, null]])"); + auto values2 = ArrayFromJSON(type, R"([[8, 9, 10], [11], null, [12]])"); + + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2}, + ArrayFromJSON(type, R"([[1, 2], null, null, null])")); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values1, values2, values1}, + ArrayFromJSON(type, R"([[1, 2], null, null, [6, null]])")); + CheckScalar("case_when", {MakeStruct({cond1, cond2}), values_null, values2, values1}, + ArrayFromJSON(type, R"([null, null, null, [6, null]])")); + } } template @@ -2351,55 +2368,123 @@ TYPED_TEST_SUITE(TestCoalesceList, ListArrowTypes); TYPED_TEST(TestCoalesceNumeric, Basics) { auto type = default_type_instance(); auto scalar_null = ScalarFromJSON(type, "null"); - auto scalar1 = ScalarFromJSON(type, "20"); auto values_null = ArrayFromJSON(type, "[null, null, null, null]"); - auto values1 = ArrayFromJSON(type, "[null, 10, 11, 12]"); - auto values2 = ArrayFromJSON(type, "[13, 14, 15, 16]"); - auto values3 = ArrayFromJSON(type, "[17, 18, 19, null]"); - // N.B. all-scalar cases are checked in CheckScalar - CheckScalar("coalesce", {values_null}, values_null); - CheckScalar("coalesce", {values_null, scalar1}, - ArrayFromJSON(type, "[20, 20, 20, 20]")); - CheckScalar("coalesce", {values_null, values1}, values1); - CheckScalar("coalesce", {values_null, values2}, values2); - CheckScalar("coalesce", {values1, values_null}, values1); - CheckScalar("coalesce", {values2, values_null}, values2); - CheckScalar("coalesce", {scalar_null, values1}, values1); - CheckScalar("coalesce", {values1, scalar_null}, values1); - CheckScalar("coalesce", {values2, values1, values_null}, values2); - CheckScalar("coalesce", {values1, scalar1}, ArrayFromJSON(type, "[20, 10, 11, 12]")); - CheckScalar("coalesce", {values1, values2}, ArrayFromJSON(type, "[13, 10, 11, 12]")); - CheckScalar("coalesce", {values1, values2, values3}, - ArrayFromJSON(type, "[13, 10, 11, 12]")); - CheckScalar("coalesce", {scalar1, values1}, ArrayFromJSON(type, "[20, 20, 20, 20]")); + + if (std::is_same::value) { + auto scalar1 = ScalarFromJSON(type, "1728000000"); + auto values1 = ArrayFromJSON(type, "[null, 864000000, 950400000, 1036800000]"); + auto values2 = + ArrayFromJSON(type, "[1123200000, 1209600000, 1296000000, 1382400000]"); + auto values3 = ArrayFromJSON(type, "[17, 18, 19, null]"); + // N.B. all-scalar cases are checked in CheckScalar + CheckScalar("coalesce", {values_null}, values_null); + CheckScalar("coalesce", {values_null, scalar1}, + ArrayFromJSON(type, "[1728000000, 1728000000, 1728000000, 1728000000]")); + CheckScalar("coalesce", {values_null, values1}, values1); + CheckScalar("coalesce", {values_null, values2}, values2); + CheckScalar("coalesce", {values1, values_null}, values1); + CheckScalar("coalesce", {values2, values_null}, values2); + CheckScalar("coalesce", {scalar_null, values1}, values1); + CheckScalar("coalesce", {values1, scalar_null}, values1); + CheckScalar("coalesce", {values2, values1, values_null}, values2); + CheckScalar("coalesce", {values1, scalar1}, + ArrayFromJSON(type, "[1728000000, 864000000, 950400000, 1036800000]")); + CheckScalar("coalesce", {values1, values2}, + ArrayFromJSON(type, "[1123200000, 864000000, 950400000, 1036800000]")); + CheckScalar("coalesce", {values1, values2, values3}, + ArrayFromJSON(type, "[1123200000, 864000000, 950400000, 1036800000]")); + CheckScalar("coalesce", {scalar1, values1}, + ArrayFromJSON(type, "[1728000000, 1728000000, 1728000000, 1728000000]")); + } else { + auto scalar1 = ScalarFromJSON(type, "20"); + auto values1 = ArrayFromJSON(type, "[null, 10, 11, 12]"); + auto values2 = ArrayFromJSON(type, "[13, 14, 15, 16]"); + auto values3 = ArrayFromJSON(type, "[17, 18, 19, null]"); + // N.B. all-scalar cases are checked in CheckScalar + CheckScalar("coalesce", {values_null}, values_null); + CheckScalar("coalesce", {values_null, scalar1}, + ArrayFromJSON(type, "[20, 20, 20, 20]")); + CheckScalar("coalesce", {values_null, values1}, values1); + CheckScalar("coalesce", {values_null, values2}, values2); + CheckScalar("coalesce", {values1, values_null}, values1); + CheckScalar("coalesce", {values2, values_null}, values2); + CheckScalar("coalesce", {scalar_null, values1}, values1); + CheckScalar("coalesce", {values1, scalar_null}, values1); + CheckScalar("coalesce", {values2, values1, values_null}, values2); + CheckScalar("coalesce", {values1, scalar1}, ArrayFromJSON(type, "[20, 10, 11, 12]")); + CheckScalar("coalesce", {values1, values2}, ArrayFromJSON(type, "[13, 10, 11, 12]")); + CheckScalar("coalesce", {values1, values2, values3}, + ArrayFromJSON(type, "[13, 10, 11, 12]")); + CheckScalar("coalesce", {scalar1, values1}, ArrayFromJSON(type, "[20, 20, 20, 20]")); + } } TYPED_TEST(TestCoalesceNumeric, ListOfType) { auto type = list(default_type_instance()); auto scalar_null = ScalarFromJSON(type, "null"); - auto scalar1 = ScalarFromJSON(type, "[20, 24]"); auto values_null = ArrayFromJSON(type, "[null, null, null, null]"); - auto values1 = ArrayFromJSON(type, "[null, [10, null, 20], [], [null, null]]"); - auto values2 = ArrayFromJSON(type, "[[23], [14, 24], [null, 15], [16]]"); - auto values3 = ArrayFromJSON(type, "[[17, 18], [19], [], null]"); - CheckScalar("coalesce", {values_null}, values_null); - CheckScalar("coalesce", {values_null, scalar1}, - ArrayFromJSON(type, "[[20, 24], [20, 24], [20, 24], [20, 24]]")); - CheckScalar("coalesce", {values_null, values1}, values1); - CheckScalar("coalesce", {values_null, values2}, values2); - CheckScalar("coalesce", {values1, values_null}, values1); - CheckScalar("coalesce", {values2, values_null}, values2); - CheckScalar("coalesce", {scalar_null, values1}, values1); - CheckScalar("coalesce", {values1, scalar_null}, values1); - CheckScalar("coalesce", {values2, values1, values_null}, values2); - CheckScalar("coalesce", {values1, scalar1}, - ArrayFromJSON(type, "[[20, 24], [10, null, 20], [], [null, null]]")); - CheckScalar("coalesce", {values1, values2}, - ArrayFromJSON(type, "[[23], [10, null, 20], [], [null, null]]")); - CheckScalar("coalesce", {values1, values2, values3}, - ArrayFromJSON(type, "[[23], [10, null, 20], [], [null, null]]")); - CheckScalar("coalesce", {scalar1, values1}, - ArrayFromJSON(type, "[[20, 24], [20, 24], [20, 24], [20, 24]]")); + + if (std::is_same::value) { + auto scalar1 = ScalarFromJSON(type, "[1728000000, 2073600000]"); + auto values1 = + ArrayFromJSON(type, "[null, [864000000, null, 1728000000], [], [null, null]]"); + auto values2 = ArrayFromJSON( + type, + "[[1987200000], [1209600000, 2073600000], [null, 1296000000], [1382400000]]"); + auto values3 = + ArrayFromJSON(type, "[[1468800000, 1555200000], [1641600000], [], null]"); + CheckScalar("coalesce", {values_null}, values_null); + CheckScalar("coalesce", {values_null, scalar1}, + ArrayFromJSON(type, + "[[1728000000, 2073600000], [1728000000, 2073600000], " + "[1728000000, 2073600000], [1728000000, 2073600000]]")); + CheckScalar("coalesce", {values_null, values1}, values1); + CheckScalar("coalesce", {values_null, values2}, values2); + CheckScalar("coalesce", {values1, values_null}, values1); + CheckScalar("coalesce", {values2, values_null}, values2); + CheckScalar("coalesce", {scalar_null, values1}, values1); + CheckScalar("coalesce", {values1, scalar_null}, values1); + CheckScalar("coalesce", {values2, values1, values_null}, values2); + CheckScalar("coalesce", {values1, scalar1}, + ArrayFromJSON(type, + "[[1728000000, 2073600000], [864000000, null, 1728000000], " + "[], [null, null]]")); + CheckScalar( + "coalesce", {values1, values2}, + ArrayFromJSON(type, + "[[1987200000], [864000000, null, 1728000000], [], [null, null]]")); + CheckScalar( + "coalesce", {values1, values2, values3}, + ArrayFromJSON(type, + "[[1987200000], [864000000, null, 1728000000], [], [null, null]]")); + CheckScalar("coalesce", {scalar1, values1}, + ArrayFromJSON(type, + "[[1728000000, 2073600000], [1728000000, 2073600000], " + "[1728000000, 2073600000], [1728000000, 2073600000]]")); + } else { + auto scalar1 = ScalarFromJSON(type, "[20, 24]"); + auto values1 = ArrayFromJSON(type, "[null, [10, null, 20], [], [null, null]]"); + auto values2 = ArrayFromJSON(type, "[[23], [14, 24], [null, 15], [16]]"); + auto values3 = ArrayFromJSON(type, "[[17, 18], [19], [], null]"); + CheckScalar("coalesce", {values_null}, values_null); + CheckScalar("coalesce", {values_null, scalar1}, + ArrayFromJSON(type, "[[20, 24], [20, 24], [20, 24], [20, 24]]")); + CheckScalar("coalesce", {values_null, values1}, values1); + CheckScalar("coalesce", {values_null, values2}, values2); + CheckScalar("coalesce", {values1, values_null}, values1); + CheckScalar("coalesce", {values2, values_null}, values2); + CheckScalar("coalesce", {scalar_null, values1}, values1); + CheckScalar("coalesce", {values1, scalar_null}, values1); + CheckScalar("coalesce", {values2, values1, values_null}, values2); + CheckScalar("coalesce", {values1, scalar1}, + ArrayFromJSON(type, "[[20, 24], [10, null, 20], [], [null, null]]")); + CheckScalar("coalesce", {values1, values2}, + ArrayFromJSON(type, "[[23], [10, null, 20], [], [null, null]]")); + CheckScalar("coalesce", {values1, values2, values3}, + ArrayFromJSON(type, "[[23], [10, null, 20], [], [null, null]]")); + CheckScalar("coalesce", {scalar1, values1}, + ArrayFromJSON(type, "[[20, 24], [20, 24], [20, 24], [20, 24]]")); + } } TYPED_TEST(TestCoalesceBinary, Basics) { From 75febd16e9afb2e220b937565859340684ba4564 Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Mon, 31 Jan 2022 15:48:29 -0500 Subject: [PATCH 15/17] Updated data generation for integration tests; other minor changes --- .../arrow/compute/kernels/vector_hash_test.cc | 8 ++--- cpp/src/arrow/testing/random.cc | 6 ++-- dev/archery/archery/integration/datagen.py | 29 +++++++++++++++++++ 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/compute/kernels/vector_hash_test.cc b/cpp/src/arrow/compute/kernels/vector_hash_test.cc index 864f9484d73..868bac1f4a6 100644 --- a/cpp/src/arrow/compute/kernels/vector_hash_test.cc +++ b/cpp/src/arrow/compute/kernels/vector_hash_test.cc @@ -162,7 +162,7 @@ TYPED_TEST(TestHashKernelPrimitive, Unique) { using T = typename TypeParam::c_type; auto type = TypeTraits::type_singleton(); - if (type->name() == "date64") { + if (type->id() == Type::DATE64) { CheckUnique( type, {172800000LL, 86400000LL, 172800000LL, 86400000LL}, {true, false, true, true}, {172800000LL, 0, 86400000LL}, {1, 0, 1}); @@ -191,7 +191,7 @@ TYPED_TEST(TestHashKernelPrimitive, ValueCounts) { using T = typename TypeParam::c_type; auto type = TypeTraits::type_singleton(); - if (type->name() == "date64") { + if (type->id() == Type::DATE64) { CheckValueCounts( type, {172800000LL, 86400000LL, 172800000LL, 86400000LL, 172800000LL, 259200000LL, @@ -225,7 +225,7 @@ TYPED_TEST(TestHashKernelPrimitive, DictEncode) { using T = typename TypeParam::c_type; auto type = TypeTraits::type_singleton(); - if (type->name() == "date64") { + if (type->id() == Type::DATE64) { CheckDictEncode( type, {172800000LL, 86400000LL, 172800000LL, 86400000LL, 172800000LL, 345600000LL}, @@ -277,7 +277,7 @@ TYPED_TEST(TestHashKernelPrimitive, PrimitiveResizeTable) { std::vector indices; std::vector counts; - if (type->name() == "date64") { + if (type->id() == Type::DATE64) { for (int64_t i = 0; i < kTotalDate64Values * kRepeats; i += kFullDayMillis) { const auto val = static_cast(i % kTotalDate64Values); values.push_back(val); diff --git a/cpp/src/arrow/testing/random.cc b/cpp/src/arrow/testing/random.cc index fb60731babb..34733ac16e7 100644 --- a/cpp/src/arrow/testing/random.cc +++ b/cpp/src/arrow/testing/random.cc @@ -803,7 +803,8 @@ std::shared_ptr RandomArrayGenerator::ArrayOf(const Field& field, int64_t } case Type::type::TIME32: { - TimeUnit::type unit = std::dynamic_pointer_cast(field.type())->unit(); + TimeUnit::type unit = + internal::checked_pointer_cast(field.type())->unit(); using c_type = typename Time32Type::c_type; const c_type min_value = 0; const c_type max_value = @@ -814,7 +815,8 @@ std::shared_ptr RandomArrayGenerator::ArrayOf(const Field& field, int64_t } case Type::type::TIME64: { - TimeUnit::type unit = std::dynamic_pointer_cast(field.type())->unit(); + TimeUnit::type unit = + internal::checked_pointer_cast(field.type())->unit(); using c_type = typename Time64Type::c_type; const c_type min_value = 0; const c_type max_value = (unit == TimeUnit::MICRO) diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py index c91d9ddf174..113197ff9ab 100644 --- a/dev/archery/archery/integration/datagen.py +++ b/dev/archery/archery/integration/datagen.py @@ -221,6 +221,30 @@ def _get_type(self): ('unit', 'DAY' if self.unit == self.DAY else 'MILLISECOND') ]) + def generate_range(self, size, lower, upper, name=None, + include_extremes=False): + if self.unit == self.DAY: + return super().generate_range(size, lower, upper, name) + + full_day_millis = 1000 * 60 * 60 * 24 + lower //= full_day_millis + upper //= full_day_millis + + values = [val * full_day_millis for val in np.random.randint( + lower, upper, size=size, dtype=np.int64)] + lower *= full_day_millis + upper *= full_day_millis + + if include_extremes and size >= 2: + values[:2] = [lower, upper] + values = list(map(int if self.bit_width < 64 else str, values)) + + is_valid = self._make_is_valid(size) + + if name is None: + name = self.name + return PrimitiveColumn(name, size, is_valid, values) + TIMEUNIT_NAMES = { 's': 'SECOND', @@ -261,6 +285,11 @@ def _get_type(self): ('bitWidth', self.bit_width) ]) + def generate_column(self, size, name=None): + lower_bound, upper_bound = self._get_generated_data_bounds() + return self.generate_range(size, lower_bound, upper_bound, + name=name) + class TimestampField(IntegerField): From e1ba73a8d43c76187dfd01d786440708deff9459 Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Mon, 31 Jan 2022 17:39:47 -0500 Subject: [PATCH 16/17] Added flag for date64 validation in integration tests, set to false --- cpp/src/arrow/testing/json_integration_test.cc | 13 +++++++++---- dev/archery/archery/integration/runner.py | 1 + dev/archery/archery/integration/tester_cpp.py | 2 ++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/testing/json_integration_test.cc b/cpp/src/arrow/testing/json_integration_test.cc index 7e1e93eae3f..b93a89dd429 100644 --- a/cpp/src/arrow/testing/json_integration_test.cc +++ b/cpp/src/arrow/testing/json_integration_test.cc @@ -60,6 +60,8 @@ DEFINE_bool( validate_decimals, true, "Validate that decimal values are in range for the given precision (ARROW-13558: " "'golden' test data from previous versions may have out-of-range decimal values)"); +DEFINE_bool(validate_date64, true, + "Validate that values for DATE64 represent whole numbers of days"); namespace arrow { @@ -126,15 +128,18 @@ static Status ConvertArrowToJson(const std::string& arrow_path, return out_file->Write(result.c_str(), static_cast(result.size())); } -// Validate the batch, accounting for the -validate_decimals flag +// Validate the batch, accounting for the -validate_decimals flag and -validate_date64 static Status ValidateFull(const RecordBatch& batch) { - if (FLAGS_validate_decimals) { + if (FLAGS_validate_decimals && FLAGS_validate_date64) { return batch.ValidateFull(); } - // Decimal validation disabled, so individually validate columns + // Decimal or date64 validation disabled, so individually validate columns RETURN_NOT_OK(batch.Validate()); for (const auto& column : batch.columns()) { - if (is_decimal(column->type()->id())) { + if (!FLAGS_validate_decimals && is_decimal(column->type()->id())) { + continue; + } + if (!FLAGS_validate_date64 && column->type()->id() == Type::DATE64) { continue; } RETURN_NOT_OK(column->ValidateFull()); diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py index 74bbed1fc4f..a2df1b33220 100644 --- a/dev/archery/archery/integration/runner.py +++ b/dev/archery/archery/integration/runner.py @@ -153,6 +153,7 @@ def _gold_tests(self, gold_dir): # ARROW-13558: older versions generated decimal values that # were out of range for the given precision. quirks.add("no_decimal_validate") + quirks.add("no_date64_validate") yield datagen.File(name, None, None, skip=skip, path=out_path, quirks=quirks) diff --git a/dev/archery/archery/integration/tester_cpp.py b/dev/archery/archery/integration/tester_cpp.py index eab520a69f0..d2ccc01ba43 100644 --- a/dev/archery/archery/integration/tester_cpp.py +++ b/dev/archery/archery/integration/tester_cpp.py @@ -60,6 +60,8 @@ def _run(self, arrow_path=None, json_path=None, command='VALIDATE', if quirks: if "no_decimal_validate" in quirks: cmd.append("--validate_decimals=false") + if "no_date64_validate" in quirks: + cmd.append("--validate_date64=false") if self.debug: log(' '.join(cmd)) From cc7192f68ccdbafa3993553ed2b41fc5daa7e73d Mon Sep 17 00:00:00 2001 From: JabariBooker Date: Tue, 1 Feb 2022 12:12:39 -0500 Subject: [PATCH 17/17] Added integration test flag for TIME32/64 types --- cpp/src/arrow/testing/json_integration_test.cc | 17 ++++++++++++----- dev/archery/archery/integration/datagen.py | 2 +- dev/archery/archery/integration/runner.py | 1 + dev/archery/archery/integration/tester_cpp.py | 2 ++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/testing/json_integration_test.cc b/cpp/src/arrow/testing/json_integration_test.cc index b93a89dd429..56c47c009c6 100644 --- a/cpp/src/arrow/testing/json_integration_test.cc +++ b/cpp/src/arrow/testing/json_integration_test.cc @@ -62,6 +62,8 @@ DEFINE_bool( "'golden' test data from previous versions may have out-of-range decimal values)"); DEFINE_bool(validate_date64, true, "Validate that values for DATE64 represent whole numbers of days"); +DEFINE_bool(validate_times, true, + "Validate that values for TIME32 and TIME64 are within their valid ranges"); namespace arrow { @@ -128,18 +130,23 @@ static Status ConvertArrowToJson(const std::string& arrow_path, return out_file->Write(result.c_str(), static_cast(result.size())); } -// Validate the batch, accounting for the -validate_decimals flag and -validate_date64 +// Validate the batch, accounting for the -validate_decimals , -validate_date64, and +// -validate_times flags static Status ValidateFull(const RecordBatch& batch) { - if (FLAGS_validate_decimals && FLAGS_validate_date64) { + if (FLAGS_validate_decimals && FLAGS_validate_date64 && FLAGS_validate_times) { return batch.ValidateFull(); } - // Decimal or date64 validation disabled, so individually validate columns + // Decimal, date64, or times32/64 validation disabled, so individually validate columns RETURN_NOT_OK(batch.Validate()); for (const auto& column : batch.columns()) { - if (!FLAGS_validate_decimals && is_decimal(column->type()->id())) { + auto type_id = column->type()->id(); + if (!FLAGS_validate_decimals && is_decimal(type_id)) { continue; } - if (!FLAGS_validate_date64 && column->type()->id() == Type::DATE64) { + if (!FLAGS_validate_date64 && type_id == Type::DATE64) { + continue; + } + if (!FLAGS_validate_times && (type_id == Type::TIME32 || type_id == Type::TIME64)) { continue; } RETURN_NOT_OK(column->ValidateFull()); diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py index 113197ff9ab..e271fdd2526 100644 --- a/dev/archery/archery/integration/datagen.py +++ b/dev/archery/archery/integration/datagen.py @@ -227,7 +227,7 @@ def generate_range(self, size, lower, upper, name=None, return super().generate_range(size, lower, upper, name) full_day_millis = 1000 * 60 * 60 * 24 - lower //= full_day_millis + lower = -1 * (abs(lower) // full_day_millis) upper //= full_day_millis values = [val * full_day_millis for val in np.random.randint( diff --git a/dev/archery/archery/integration/runner.py b/dev/archery/archery/integration/runner.py index a2df1b33220..b67f47e2ce9 100644 --- a/dev/archery/archery/integration/runner.py +++ b/dev/archery/archery/integration/runner.py @@ -154,6 +154,7 @@ def _gold_tests(self, gold_dir): # were out of range for the given precision. quirks.add("no_decimal_validate") quirks.add("no_date64_validate") + quirks.add("no_times_validate") yield datagen.File(name, None, None, skip=skip, path=out_path, quirks=quirks) diff --git a/dev/archery/archery/integration/tester_cpp.py b/dev/archery/archery/integration/tester_cpp.py index d2ccc01ba43..4e74576ede8 100644 --- a/dev/archery/archery/integration/tester_cpp.py +++ b/dev/archery/archery/integration/tester_cpp.py @@ -62,6 +62,8 @@ def _run(self, arrow_path=None, json_path=None, command='VALIDATE', cmd.append("--validate_decimals=false") if "no_date64_validate" in quirks: cmd.append("--validate_date64=false") + if "no_times_validate" in quirks: + cmd.append("--validate_times=false") if self.debug: log(' '.join(cmd))