Skip to content

ARROW-16865: [C++][Python] Implement cumulative product, max, and min compute functions #13412

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
JabariBooker wants to merge 14 commits into from
Closed

ARROW-16865: [C++][Python] Implement cumulative product, max, and min compute functions #13412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
c664f8e
Adding CumulativeProduct compute function
JabariBooker Jun 21, 2022
b7605dd
Correcting linting errors
JabariBooker Jun 21, 2022
e197309
Added CumulativeMin and CumulativeMax compute functions
JabariBooker Jun 27, 2022
6e451ee
Missing variable in CumulativeSum and -Product
JabariBooker Jun 27, 2022
1060884
Updating docs for CumulativeProduct, -Min, and -Max
JabariBooker Jul 1, 2022
122dad8
Adding tests for CumulativeProduct
JabariBooker Jul 5, 2022
f560d2d
Adding the rest of tests for CumulativeProduct
JabariBooker Jul 5, 2022
7987b90
Added tests for CumulativeMin and -Max
JabariBooker Jul 5, 2022
29176e1
Added pytests for CumulativeProduct, -Min, and -Max
JabariBooker Jul 5, 2022
1036779
Forgot linting issue
JabariBooker Jul 5, 2022
e81cedd
Merge branch 'master' into ARROW-16865
JabariBooker Jul 11, 2022
2e6527d
Completed C++ and Python unit tests
JabariBooker Jul 28, 2022
cb4a4cd
Correcting linting
JabariBooker Jul 28, 2022
858cadd
Removing unused math.isnan import
JabariBooker Jul 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 67 additions & 2 deletions cpp/src/arrow/compute/api_vector.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,17 @@ static auto kCumulativeSumOptionsType = GetFunctionOptionsType<CumulativeSumOpti
DataMember("start", &CumulativeSumOptions::start),
DataMember("skip_nulls", &CumulativeSumOptions::skip_nulls),
DataMember("check_overflow", &CumulativeSumOptions::check_overflow));
static auto kCumulativeProductOptionsType =
GetFunctionOptionsType<CumulativeProductOptions>(
DataMember("start", &CumulativeProductOptions::start),
DataMember("skip_nulls", &CumulativeProductOptions::skip_nulls),
DataMember("check_overflow", &CumulativeProductOptions::check_overflow));
static auto kCumulativeMinOptionsType = GetFunctionOptionsType<CumulativeMinOptions>(
DataMember("start", &CumulativeMinOptions::start),
DataMember("skip_nulls", &CumulativeMinOptions::skip_nulls));
static auto kCumulativeMaxOptionsType = GetFunctionOptionsType<CumulativeMaxOptions>(
DataMember("start", &CumulativeMaxOptions::start),
DataMember("skip_nulls", &CumulativeMaxOptions::skip_nulls));
static auto kRankOptionsType = GetFunctionOptionsType<RankOptions>(
DataMember("sort_keys", &RankOptions::sort_keys),
DataMember("null_placement", &RankOptions::null_placement),
Expand Down Expand Up @@ -218,6 +229,38 @@ CumulativeSumOptions::CumulativeSumOptions(std::shared_ptr<Scalar> start, bool s
check_overflow(check_overflow) {}
constexpr char CumulativeSumOptions::kTypeName[];

CumulativeProductOptions::CumulativeProductOptions(double start, bool skip_nulls,
bool check_overflow)
: CumulativeProductOptions(std::make_shared<DoubleScalar>(start), skip_nulls,
check_overflow) {}
CumulativeProductOptions::CumulativeProductOptions(std::shared_ptr<Scalar> start,
bool skip_nulls, bool check_overflow)
: FunctionOptions(internal::kCumulativeProductOptionsType),
start(std::move(start)),
skip_nulls(skip_nulls),
check_overflow(check_overflow) {}
constexpr char CumulativeProductOptions::kTypeName[];

CumulativeMinOptions::CumulativeMinOptions(bool skip_nulls)
: FunctionOptions(internal::kCumulativeMinOptionsType), skip_nulls(skip_nulls) {}
CumulativeMinOptions::CumulativeMinOptions(double start, bool skip_nulls)
: CumulativeMinOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeMinOptions::CumulativeMinOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
: FunctionOptions(internal::kCumulativeMinOptionsType),
start(std::move(start)),
skip_nulls(skip_nulls) {}
constexpr char CumulativeMinOptions::kTypeName[];

CumulativeMaxOptions::CumulativeMaxOptions(bool skip_nulls)
: FunctionOptions(internal::kCumulativeMaxOptionsType), skip_nulls(skip_nulls) {}
CumulativeMaxOptions::CumulativeMaxOptions(double start, bool skip_nulls)
: CumulativeMaxOptions(std::make_shared<DoubleScalar>(start), skip_nulls) {}
CumulativeMaxOptions::CumulativeMaxOptions(std::shared_ptr<Scalar> start, bool skip_nulls)
: FunctionOptions(internal::kCumulativeMaxOptionsType),
start(std::move(start)),
skip_nulls(skip_nulls) {}
constexpr char CumulativeMaxOptions::kTypeName[];

RankOptions::RankOptions(std::vector<SortKey> sort_keys, NullPlacement null_placement,
RankOptions::Tiebreaker tiebreaker)
: FunctionOptions(internal::kRankOptionsType),
Expand All @@ -236,6 +279,9 @@ void RegisterVectorOptions(FunctionRegistry* registry) {
DCHECK_OK(registry->AddFunctionOptionsType(kPartitionNthOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kSelectKOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeSumOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeProductOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeMinOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kCumulativeMaxOptionsType));
DCHECK_OK(registry->AddFunctionOptionsType(kRankOptionsType));
}
} // namespace internal
Expand Down Expand Up @@ -379,8 +425,27 @@ Result<std::shared_ptr<Array>> DropNull(const Array& values, ExecContext* ctx) {

Result<Datum> CumulativeSum(const Datum& values, const CumulativeSumOptions& options,
ExecContext* ctx) {
auto func_name = (options.check_overflow) ? "cumulative_sum_checked" : "cumulative_sum";
return CallFunction(func_name, {Datum(values)}, &options, ctx);
return CallFunction(
options.check_overflow ? "cumulative_sum_checked" : "cumulative_sum",
{Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeProduct(const Datum& values,
const CumulativeProductOptions& options,
ExecContext* ctx) {
return CallFunction(
options.check_overflow ? "cumulative_product_checked" : "cumulative_product",
{Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMin(const Datum& values, const CumulativeMinOptions& options,
ExecContext* ctx) {
return CallFunction("cumulative_min", {Datum(values)}, &options, ctx);
}

Result<Datum> CumulativeMax(const Datum& values, const CumulativeMaxOptions& options,
ExecContext* ctx) {
return CallFunction("cumulative_max", {Datum(values)}, &options, ctx);
}

// ----------------------------------------------------------------------
Expand Down
87 changes: 86 additions & 1 deletion cpp/src/arrow/compute/api_vector.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,10 @@ class ARROW_EXPORT CumulativeSumOptions : public FunctionOptions {
static constexpr char const kTypeName[] = "CumulativeSumOptions";
static CumulativeSumOptions Defaults() { return CumulativeSumOptions(); }

/// Optional starting value for cumulative operation computation
const bool is_minmax = false;
const bool is_max = false;

/// Optional starting value for cumulative sum
std::shared_ptr<Scalar> start;

/// If true, nulls in the input are ignored and produce a corresponding null output.
Expand All @@ -246,6 +249,70 @@ class ARROW_EXPORT CumulativeSumOptions : public FunctionOptions {
bool check_overflow = false;
};

/// \brief Options for cumulative product function
class ARROW_EXPORT CumulativeProductOptions : public FunctionOptions {
public:
explicit CumulativeProductOptions(double start = 1, bool skip_nulls = false,
bool check_overflow = false);
explicit CumulativeProductOptions(std::shared_ptr<Scalar> start,
bool skip_nulls = false, bool check_overflow = false);
static constexpr char const kTypeName[] = "CumulativeProductOptions";
static CumulativeProductOptions Defaults() { return CumulativeProductOptions(); }

const bool is_minmax = false;
const bool is_max = false;

/// Optional starting value for cumulative product
std::shared_ptr<Scalar> start;

/// If true, nulls in the input are ignored and produce a corresponding null output.
/// When false, the first null encountered is propagated through the remaining output.
bool skip_nulls = false;

/// When true, returns an Invalid Status when overflow is detected
bool check_overflow = false;
};

/// \brief Options for cumulative min functions
class ARROW_EXPORT CumulativeMinOptions : public FunctionOptions {
public:
explicit CumulativeMinOptions(bool skip_nulls = false);
explicit CumulativeMinOptions(double start, bool skip_nulls = false);
explicit CumulativeMinOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
static constexpr char const kTypeName[] = "CumulativeMinOptions";
static CumulativeMinOptions Defaults() { return CumulativeMinOptions(); }

const bool is_minmax = true;
Copy link
Member

@lidavidm lidavidm Aug 9, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: it might be more idiomatic to have this as constexpr static bool, then below use if (OptionsType::is_minmax)

const bool is_max = false;

/// Optional starting value for cumulative min
std::shared_ptr<Scalar> start;

/// If true, nulls in the input are ignored and produce a corresponding null output.
/// When false, the first null encountered is propagated through the remaining output.
bool skip_nulls = false;
};

/// \brief Options for cumulative max functions
class ARROW_EXPORT CumulativeMaxOptions : public FunctionOptions {
public:
explicit CumulativeMaxOptions(bool skip_nulls = false);
explicit CumulativeMaxOptions(double start, bool skip_nulls = false);
explicit CumulativeMaxOptions(std::shared_ptr<Scalar> start, bool skip_nulls = false);
static constexpr char const kTypeName[] = "CumulativeMaxOptions";
static CumulativeMaxOptions Defaults() { return CumulativeMaxOptions(); }

const bool is_minmax = true;
const bool is_max = true;

/// Optional starting value for cumulative max
std::shared_ptr<Scalar> start;

/// If true, nulls in the input are ignored and produce a corresponding null output.
/// When false, the first null encountered is propagated through the remaining output.
bool skip_nulls = false;
};

/// @}

/// \brief Filter with a boolean selection filter
Expand Down Expand Up @@ -586,6 +653,24 @@ Result<Datum> CumulativeSum(
const CumulativeSumOptions& options = CumulativeSumOptions::Defaults(),
ExecContext* ctx = NULLPTR);

ARROW_EXPORT
Result<Datum> CumulativeProduct(
const Datum& values,
const CumulativeProductOptions& options = CumulativeProductOptions::Defaults(),
ExecContext* ctx = NULLPTR);

ARROW_EXPORT
Result<Datum> CumulativeMin(
const Datum& values,
const CumulativeMinOptions& options = CumulativeMinOptions::Defaults(),
ExecContext* ctx = NULLPTR);

ARROW_EXPORT
Result<Datum> CumulativeMax(
const Datum& values,
const CumulativeMaxOptions& options = CumulativeMaxOptions::Defaults(),
ExecContext* ctx = NULLPTR);

// ----------------------------------------------------------------------
// Deprecated functions

Expand Down
49 changes: 49 additions & 0 deletions cpp/src/arrow/compute/kernels/base_arithmetic_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -597,6 +597,55 @@ struct Sign {
}
};

struct Min {
template <typename T, typename Arg0, typename Arg1>
static constexpr T Call(KernelContext*, Arg0 left, Arg1 right, Status*) {
return (left < right) ? left : right;
}
};
struct Max {
template <typename T, typename Arg0, typename Arg1>
static constexpr T Call(KernelContext*, Arg0 left, Arg1 right, Status*) {
return (left > right) ? left : right;
}
};

template <typename CType>
struct AntiExtrema {
static constexpr CType anti_min() { return std::numeric_limits<CType>::max(); }
static constexpr CType anti_max() { return std::numeric_limits<CType>::min(); }
};

template <>
struct AntiExtrema<bool> {
static constexpr bool anti_min() { return true; }
static constexpr bool anti_max() { return false; }
};

template <>
struct AntiExtrema<float> {
static constexpr float anti_min() { return std::numeric_limits<float>::infinity(); }
static constexpr float anti_max() { return -std::numeric_limits<float>::infinity(); }
};

template <>
struct AntiExtrema<double> {
static constexpr double anti_min() { return std::numeric_limits<double>::infinity(); }
static constexpr double anti_max() { return -std::numeric_limits<double>::infinity(); }
};

template <>
struct AntiExtrema<Decimal128> {
static constexpr Decimal128 anti_min() { return BasicDecimal128::GetMaxSentinel(); }
static constexpr Decimal128 anti_max() { return BasicDecimal128::GetMinSentinel(); }
};

template <>
struct AntiExtrema<Decimal256> {
static constexpr Decimal256 anti_min() { return BasicDecimal256::GetMaxSentinel(); }
static constexpr Decimal256 anti_max() { return BasicDecimal256::GetMinSentinel(); }
};

} // namespace internal
} // namespace compute
} // namespace arrow
38 changes: 1 addition & 37 deletions cpp/src/arrow/compute/kernels/hash_aggregate.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#include "arrow/compute/kernel.h"
#include "arrow/compute/kernels/aggregate_internal.h"
#include "arrow/compute/kernels/aggregate_var_std_internal.h"
#include "arrow/compute/kernels/base_arithmetic_internal.h"
#include "arrow/compute/kernels/common.h"
#include "arrow/compute/kernels/row_encoder.h"
#include "arrow/compute/kernels/util_internal.h"
Expand Down Expand Up @@ -1199,43 +1200,6 @@ HashAggregateKernel MakeApproximateMedianKernel(HashAggregateFunction* tdigest_f

// ----------------------------------------------------------------------
// MinMax implementation

template <typename CType>
struct AntiExtrema {
static constexpr CType anti_min() { return std::numeric_limits<CType>::max(); }
static constexpr CType anti_max() { return std::numeric_limits<CType>::min(); }
};

template <>
struct AntiExtrema<bool> {
static constexpr bool anti_min() { return true; }
static constexpr bool anti_max() { return false; }
};

template <>
struct AntiExtrema<float> {
static constexpr float anti_min() { return std::numeric_limits<float>::infinity(); }
static constexpr float anti_max() { return -std::numeric_limits<float>::infinity(); }
};

template <>
struct AntiExtrema<double> {
static constexpr double anti_min() { return std::numeric_limits<double>::infinity(); }
static constexpr double anti_max() { return -std::numeric_limits<double>::infinity(); }
};

template <>
struct AntiExtrema<Decimal128> {
static constexpr Decimal128 anti_min() { return BasicDecimal128::GetMaxSentinel(); }
static constexpr Decimal128 anti_max() { return BasicDecimal128::GetMinSentinel(); }
};

template <>
struct AntiExtrema<Decimal256> {
static constexpr Decimal256 anti_min() { return BasicDecimal256::GetMaxSentinel(); }
static constexpr Decimal256 anti_max() { return BasicDecimal256::GetMinSentinel(); }
};

template <typename Type, typename Enable = void>
struct GroupedMinMaxImpl final : public GroupedAggregator {
using CType = typename TypeTraits<Type>::CType;
Expand Down
Loading