diff --git a/cpp/src/arrow/array/array_base.cc b/cpp/src/arrow/array/array_base.cc index 67c5ca84e1f..682bc781cf2 100644 --- a/cpp/src/arrow/array/array_base.cc +++ b/cpp/src/arrow/array/array_base.cc @@ -69,13 +69,18 @@ struct ScalarFromArraySlotImpl { return Finish(a.Value(index_)); } - Status Visit(const Decimal128Array& a) { - return Finish(Decimal128(a.GetValue(index_))); +#define DECL_DECIMAL_VISIT(width) \ + Status Visit(const Decimal##width##Array& a) { \ + return Finish(Decimal##width(a.GetValue(index_))); \ } - Status Visit(const Decimal256Array& a) { - return Finish(Decimal256(a.GetValue(index_))); - } + DECL_DECIMAL_VISIT(16) + DECL_DECIMAL_VISIT(32) + DECL_DECIMAL_VISIT(64) + DECL_DECIMAL_VISIT(128) + DECL_DECIMAL_VISIT(256) + +#undef DECL_DECIMAL_VISIT template Status Visit(const BaseBinaryArray& a) { diff --git a/cpp/src/arrow/array/array_decimal.cc b/cpp/src/arrow/array/array_decimal.cc index d65f6ee5356..3d0187bd2eb 100644 --- a/cpp/src/arrow/array/array_decimal.cc +++ b/cpp/src/arrow/array/array_decimal.cc @@ -32,32 +32,23 @@ namespace arrow { using internal::checked_cast; -// ---------------------------------------------------------------------- -// Decimal128 - -Decimal128Array::Decimal128Array(const std::shared_ptr& data) +template +BaseDecimalArray::BaseDecimalArray(const std::shared_ptr& data) : FixedSizeBinaryArray(data) { - ARROW_CHECK_EQ(data->type->id(), Type::DECIMAL128); + ARROW_CHECK_EQ(data->type->id(), DecimalTypeTraits::Id); } -std::string Decimal128Array::FormatValue(int64_t i) const { - const auto& type_ = checked_cast(*type()); - const Decimal128 value(GetValue(i)); +template +std::string BaseDecimalArray::FormatValue(int64_t i) const { + const auto& type_ = checked_cast(*type()); + const ValueType value(GetValue(i)); return value.ToString(type_.scale()); } -// ---------------------------------------------------------------------- -// Decimal256 - -Decimal256Array::Decimal256Array(const std::shared_ptr& data) - : FixedSizeBinaryArray(data) { - ARROW_CHECK_EQ(data->type->id(), Type::DECIMAL256); -} - -std::string Decimal256Array::FormatValue(int64_t i) const { - const auto& type_ = checked_cast(*type()); - const Decimal256 value(GetValue(i)); - return value.ToString(type_.scale()); -} +template class ARROW_EXPORT BaseDecimalArray<16>; +template class ARROW_EXPORT BaseDecimalArray<32>; +template class ARROW_EXPORT BaseDecimalArray<64>; +template class ARROW_EXPORT BaseDecimalArray<128>; +template class ARROW_EXPORT BaseDecimalArray<256>; } // namespace arrow diff --git a/cpp/src/arrow/array/array_decimal.h b/cpp/src/arrow/array/array_decimal.h index 8d7d1c59cd0..0f8d8101912 100644 --- a/cpp/src/arrow/array/array_decimal.h +++ b/cpp/src/arrow/array/array_decimal.h @@ -24,43 +24,27 @@ #include "arrow/array/array_binary.h" #include "arrow/array/data.h" #include "arrow/type.h" +#include "arrow/util/decimal_type_traits.h" #include "arrow/util/visibility.h" namespace arrow { -// ---------------------------------------------------------------------- -// Decimal128Array - -/// Concrete Array class for 128-bit decimal data -class ARROW_EXPORT Decimal128Array : public FixedSizeBinaryArray { +/// Template Array class for decimal data +template +class ARROW_EXPORT BaseDecimalArray : public FixedSizeBinaryArray { public: - using TypeClass = Decimal128Type; + using TypeClass = typename DecimalTypeTraits::TypeClass; + using ValueType = typename DecimalTypeTraits::ValueType; using FixedSizeBinaryArray::FixedSizeBinaryArray; - /// \brief Construct Decimal128Array from ArrayData instance - explicit Decimal128Array(const std::shared_ptr& data); + /// \brief Construct DecimalArray from ArrayData instance + explicit BaseDecimalArray(const std::shared_ptr& data); std::string FormatValue(int64_t i) const; }; // Backward compatibility -using DecimalArray = Decimal128Array; - -// ---------------------------------------------------------------------- -// Decimal256Array - -/// Concrete Array class for 256-bit decimal data -class ARROW_EXPORT Decimal256Array : public FixedSizeBinaryArray { - public: - using TypeClass = Decimal256Type; - - using FixedSizeBinaryArray::FixedSizeBinaryArray; - - /// \brief Construct Decimal256Array from ArrayData instance - explicit Decimal256Array(const std::shared_ptr& data); - - std::string FormatValue(int64_t i) const; -}; +using DecimalArray = BaseDecimalArray<128>; } // namespace arrow diff --git a/cpp/src/arrow/array/array_dict_test.cc b/cpp/src/arrow/array/array_dict_test.cc index fca442b2567..2bc36a82d11 100644 --- a/cpp/src/arrow/array/array_dict_test.cc +++ b/cpp/src/arrow/array/array_dict_test.cc @@ -859,6 +859,18 @@ void TestDecimalDictionaryBuilderBasic(std::shared_ptr decimal_type) { ASSERT_TRUE(expected.Equals(result)); } +TEST(TestDecimal16DictionaryBuilder, Basic) { + TestDecimalDictionaryBuilderBasic(arrow::decimal16(2, 0)); +} + +TEST(TestDecimal32DictionaryBuilder, Basic) { + TestDecimalDictionaryBuilderBasic(arrow::decimal32(2, 0)); +} + +TEST(TestDecimal64DictionaryBuilder, Basic) { + TestDecimalDictionaryBuilderBasic(arrow::decimal64(2, 0)); +} + TEST(TestDecimal128DictionaryBuilder, Basic) { TestDecimalDictionaryBuilderBasic(arrow::decimal128(2, 0)); } diff --git a/cpp/src/arrow/array/array_test.cc b/cpp/src/arrow/array/array_test.cc index 89087ee318c..0f9b8fda85e 100644 --- a/cpp/src/arrow/array/array_test.cc +++ b/cpp/src/arrow/array/array_test.cc @@ -2407,6 +2407,34 @@ class DecimalTest : public ::testing::TestWithParam { } } + void InitNoNullsTest(int32_t precision) { + std::vector draw = {DecimalValue(1), DecimalValue(-2), + DecimalValue(2389), DecimalValue(4), + DecimalValue(-12348)}; + std::vector valid_bytes = {true, true, true, true, true}; + this->TestCreate(precision, draw, valid_bytes, 0); + this->TestCreate(precision, draw, valid_bytes, 2); + } + + void InitWithNullsTest(int32_t precision, std::string big_value, + std::string big_negate_value) { + std::vector draw = {DecimalValue(1), DecimalValue(2), DecimalValue(-1), + DecimalValue(4), DecimalValue(-1), DecimalValue(1), + DecimalValue(2)}; + DecimalValue big; + ASSERT_OK_AND_ASSIGN(big, DecimalValue::FromString(big_value)); + draw.push_back(big); + + DecimalValue big_negative; + ASSERT_OK_AND_ASSIGN(big_negative, DecimalValue::FromString(big_negate_value)); + draw.push_back(big_negative); + + std::vector valid_bytes = {true, true, false, true, false, + true, true, true, true}; + this->TestCreate(precision, draw, valid_bytes, 0); + this->TestCreate(precision, draw, valid_bytes, 2); + } + template void TestCreate(int32_t precision, const DecimalVector& draw, const std::vector& valid_bytes, int64_t offset) const { @@ -2451,34 +2479,58 @@ class DecimalTest : public ::testing::TestWithParam { } }; +using Decimal16Test = DecimalTest; + +TEST_P(Decimal16Test, NoNulls) { + int32_t precision = GetParam(); + this->InitNoNullsTest(precision); +} + +TEST_P(Decimal16Test, WithNulls) { + int32_t precision = GetParam(); + this->InitWithNullsTest(precision, "163.84", "-163.84"); +} + +INSTANTIATE_TEST_SUITE_P(Decimal16Test, Decimal16Test, ::testing::Range(1, 5)); + +using Decimal32Test = DecimalTest; + +TEST_P(Decimal32Test, NoNulls) { + int32_t precision = GetParam(); + this->InitNoNullsTest(precision); +} + +TEST_P(Decimal32Test, WithNulls) { + int32_t precision = GetParam(); + this->InitWithNullsTest(precision, "107374.1824", "-107374.1824"); +} + +INSTANTIATE_TEST_SUITE_P(Decimal32Test, Decimal32Test, ::testing::Range(1, 10)); + +using Decimal64Test = DecimalTest; + +TEST_P(Decimal64Test, NoNulls) { + int32_t precision = GetParam(); + this->InitNoNullsTest(precision); +} + +TEST_P(Decimal64Test, WithNulls) { + int32_t precision = GetParam(); + this->InitWithNullsTest(precision, "46116860184.27387904", "-46116860184.27387904"); +} + +INSTANTIATE_TEST_SUITE_P(Decimal64Test, Decimal64Test, ::testing::Range(1, 19)); + using Decimal128Test = DecimalTest; TEST_P(Decimal128Test, NoNulls) { int32_t precision = GetParam(); - std::vector draw = {Decimal128(1), Decimal128(-2), Decimal128(2389), - Decimal128(4), Decimal128(-12348)}; - std::vector valid_bytes = {true, true, true, true, true}; - this->TestCreate(precision, draw, valid_bytes, 0); - this->TestCreate(precision, draw, valid_bytes, 2); + this->InitNoNullsTest(precision); } TEST_P(Decimal128Test, WithNulls) { int32_t precision = GetParam(); - std::vector draw = {Decimal128(1), Decimal128(2), Decimal128(-1), - Decimal128(4), Decimal128(-1), Decimal128(1), - Decimal128(2)}; - Decimal128 big; - ASSERT_OK_AND_ASSIGN(big, Decimal128::FromString("230342903942.234234")); - draw.push_back(big); - - Decimal128 big_negative; - ASSERT_OK_AND_ASSIGN(big_negative, Decimal128::FromString("-23049302932.235234")); - draw.push_back(big_negative); - - std::vector valid_bytes = {true, true, false, true, false, - true, true, true, true}; - this->TestCreate(precision, draw, valid_bytes, 0); - this->TestCreate(precision, draw, valid_bytes, 2); + this->InitWithNullsTest(precision, "23049302932.235234", "-23049302932.235234"); } INSTANTIATE_TEST_SUITE_P(Decimal128Test, Decimal128Test, ::testing::Range(1, 38)); @@ -2487,34 +2539,16 @@ using Decimal256Test = DecimalTest; TEST_P(Decimal256Test, NoNulls) { int32_t precision = GetParam(); - std::vector draw = {Decimal256(1), Decimal256(-2), Decimal256(2389), - Decimal256(4), Decimal256(-12348)}; - std::vector valid_bytes = {true, true, true, true, true}; - this->TestCreate(precision, draw, valid_bytes, 0); - this->TestCreate(precision, draw, valid_bytes, 2); + this->InitNoNullsTest(precision); } TEST_P(Decimal256Test, WithNulls) { int32_t precision = GetParam(); - std::vector draw = {Decimal256(1), Decimal256(2), Decimal256(-1), - Decimal256(4), Decimal256(-1), Decimal256(1), - Decimal256(2)}; - Decimal256 big; // (pow(2, 255) - 1) / pow(10, 38) - ASSERT_OK_AND_ASSIGN(big, - Decimal256::FromString("578960446186580977117854925043439539266." - "34992332820282019728792003956564819967")); - draw.push_back(big); - - Decimal256 big_negative; // -pow(2, 255) / pow(10, 38) - ASSERT_OK_AND_ASSIGN(big_negative, - Decimal256::FromString("-578960446186580977117854925043439539266." - "34992332820282019728792003956564819968")); - draw.push_back(big_negative); - - std::vector valid_bytes = {true, true, false, true, false, - true, true, true, true}; - this->TestCreate(precision, draw, valid_bytes, 0); - this->TestCreate(precision, draw, valid_bytes, 2); + this->InitWithNullsTest(precision, + "578960446186580977117854925043439539266." + "34992332820282019728792003956564819967", + "-578960446186580977117854925043439539266." + "34992332820282019728792003956564819968"); } INSTANTIATE_TEST_SUITE_P(Decimal256Test, Decimal256Test, diff --git a/cpp/src/arrow/array/builder_decimal.cc b/cpp/src/arrow/array/builder_decimal.cc index bd7615a7309..19e5ddd8b5a 100644 --- a/cpp/src/arrow/array/builder_decimal.cc +++ b/cpp/src/arrow/array/builder_decimal.cc @@ -33,30 +33,35 @@ class Buffer; class MemoryPool; // ---------------------------------------------------------------------- -// Decimal128Builder +// BaseDecimalBuilder -Decimal128Builder::Decimal128Builder(const std::shared_ptr& type, - MemoryPool* pool) +template +BaseDecimalBuilder::BaseDecimalBuilder(const std::shared_ptr& type, + MemoryPool* pool) : FixedSizeBinaryBuilder(type, pool), - decimal_type_(internal::checked_pointer_cast(type)) {} + decimal_type_(internal::checked_pointer_cast(type)) {} -Status Decimal128Builder::Append(Decimal128 value) { +template +Status BaseDecimalBuilder::Append(ValueType value) { RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1)); UnsafeAppend(value); return Status::OK(); } -void Decimal128Builder::UnsafeAppend(Decimal128 value) { +template +void BaseDecimalBuilder::UnsafeAppend(ValueType value) { value.ToBytes(GetMutableValue(length())); - byte_builder_.UnsafeAdvance(16); + byte_builder_.UnsafeAdvance((width >> 3)); UnsafeAppendToBitmap(true); } -void Decimal128Builder::UnsafeAppend(util::string_view value) { +template +void BaseDecimalBuilder::UnsafeAppend(util::string_view value) { FixedSizeBinaryBuilder::UnsafeAppend(value); } -Status Decimal128Builder::FinishInternal(std::shared_ptr* out) { +template +Status BaseDecimalBuilder::FinishInternal(std::shared_ptr* out) { std::shared_ptr data; RETURN_NOT_OK(byte_builder_.Finish(&data)); std::shared_ptr null_bitmap; @@ -67,39 +72,10 @@ Status Decimal128Builder::FinishInternal(std::shared_ptr* out) { return Status::OK(); } -// ---------------------------------------------------------------------- -// Decimal256Builder - -Decimal256Builder::Decimal256Builder(const std::shared_ptr& type, - MemoryPool* pool) - : FixedSizeBinaryBuilder(type, pool), - decimal_type_(internal::checked_pointer_cast(type)) {} - -Status Decimal256Builder::Append(const Decimal256& value) { - RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1)); - UnsafeAppend(value); - return Status::OK(); -} - -void Decimal256Builder::UnsafeAppend(const Decimal256& value) { - value.ToBytes(GetMutableValue(length())); - byte_builder_.UnsafeAdvance(32); - UnsafeAppendToBitmap(true); -} - -void Decimal256Builder::UnsafeAppend(util::string_view value) { - FixedSizeBinaryBuilder::UnsafeAppend(value); -} - -Status Decimal256Builder::FinishInternal(std::shared_ptr* out) { - std::shared_ptr data; - RETURN_NOT_OK(byte_builder_.Finish(&data)); - std::shared_ptr null_bitmap; - RETURN_NOT_OK(null_bitmap_builder_.Finish(&null_bitmap)); - - *out = ArrayData::Make(type(), length_, {null_bitmap, data}, null_count_); - capacity_ = length_ = null_count_ = 0; - return Status::OK(); -} +template class ARROW_EXPORT BaseDecimalBuilder<16>; +template class ARROW_EXPORT BaseDecimalBuilder<32>; +template class ARROW_EXPORT BaseDecimalBuilder<64>; +template class ARROW_EXPORT BaseDecimalBuilder<128>; +template class ARROW_EXPORT BaseDecimalBuilder<256>; } // namespace arrow diff --git a/cpp/src/arrow/array/builder_decimal.h b/cpp/src/arrow/array/builder_decimal.h index 8c75e7dd674..a04dfd415fd 100644 --- a/cpp/src/arrow/array/builder_decimal.h +++ b/cpp/src/arrow/array/builder_decimal.h @@ -25,23 +25,27 @@ #include "arrow/array/data.h" #include "arrow/status.h" #include "arrow/type.h" +#include "arrow/util/decimal_type_traits.h" #include "arrow/util/visibility.h" namespace arrow { -class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder { +template +class ARROW_EXPORT BaseDecimalBuilder : public FixedSizeBinaryBuilder { public: - using TypeClass = Decimal128Type; + using TypeClass = typename DecimalTypeTraits::TypeClass; + using ArrayType = typename DecimalTypeTraits::ArrayType; + using ValueType = typename DecimalTypeTraits::ValueType; - explicit Decimal128Builder(const std::shared_ptr& type, - MemoryPool* pool = default_memory_pool()); + explicit BaseDecimalBuilder(const std::shared_ptr& type, + MemoryPool* pool = default_memory_pool()); using FixedSizeBinaryBuilder::Append; using FixedSizeBinaryBuilder::AppendValues; using FixedSizeBinaryBuilder::Reset; - Status Append(Decimal128 val); - void UnsafeAppend(Decimal128 val); + Status Append(ValueType val); + void UnsafeAppend(ValueType val); void UnsafeAppend(util::string_view val); Status FinishInternal(std::shared_ptr* out) override; @@ -50,43 +54,15 @@ class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder { using ArrayBuilder::Finish; /// \endcond - Status Finish(std::shared_ptr* out) { return FinishTyped(out); } + Status Finish(std::shared_ptr* out) { return FinishTyped(out); } std::shared_ptr type() const override { return decimal_type_; } protected: - std::shared_ptr decimal_type_; + std::shared_ptr decimal_type_; }; -class ARROW_EXPORT Decimal256Builder : public FixedSizeBinaryBuilder { - public: - using TypeClass = Decimal256Type; - - explicit Decimal256Builder(const std::shared_ptr& type, - MemoryPool* pool = default_memory_pool()); - - using FixedSizeBinaryBuilder::Append; - using FixedSizeBinaryBuilder::AppendValues; - using FixedSizeBinaryBuilder::Reset; - - Status Append(const Decimal256& val); - void UnsafeAppend(const Decimal256& val); - void UnsafeAppend(util::string_view val); - - Status FinishInternal(std::shared_ptr* out) override; - - /// \cond FALSE - using ArrayBuilder::Finish; - /// \endcond - - Status Finish(std::shared_ptr* out) { return FinishTyped(out); } - - std::shared_ptr type() const override { return decimal_type_; } - - protected: - std::shared_ptr decimal_type_; -}; - -using DecimalBuilder = Decimal128Builder; +// Backward compatibility +using DecimalBuilder = BaseDecimalBuilder<128>; } // namespace arrow diff --git a/cpp/src/arrow/array/builder_dict.h b/cpp/src/arrow/array/builder_dict.h index 40d6ce1ba9a..131f256c992 100644 --- a/cpp/src/arrow/array/builder_dict.h +++ b/cpp/src/arrow/array/builder_dict.h @@ -238,6 +238,30 @@ class DictionaryBuilderBase : public ArrayBuilder { return Append(util::string_view(value, length)); } + /// \brief Append a decimal (only for Decimal16Type) + template + enable_if_decimal16 Append(const Decimal16& value) { + uint8_t data[2]; + value.ToBytes(data); + return Append(data, 2); + } + + /// \brief Append a decimal (only for Decimal32Type) + template + enable_if_decimal32 Append(const Decimal32& value) { + uint8_t data[4]; + value.ToBytes(data); + return Append(data, 4); + } + + /// \brief Append a decimal (only for Decimal64Type) + template + enable_if_decimal64 Append(const Decimal64& value) { + uint8_t data[8]; + value.ToBytes(data); + return Append(data, 8); + } + /// \brief Append a decimal (only for Decimal128Type) template enable_if_decimal128 Append(const Decimal128& value) { diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc index 38092045aab..47f5309c3c8 100644 --- a/cpp/src/arrow/array/validate.cc +++ b/cpp/src/arrow/array/validate.cc @@ -62,6 +62,14 @@ struct ValidateArrayImpl { return Status::OK(); } + template + Status Visit(const BaseDecimalArray& array) { + if (array.length() > 0 && array.values() == nullptr) { + return Status::Invalid("values is null"); + } + return Status::OK(); + } + Status Visit(const StringType& type) { return ValidateBinaryLike(type); } Status Visit(const BinaryType& type) { return ValidateBinaryLike(type); } diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index f22228a4588..e3b5a699712 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -50,6 +50,9 @@ struct DictionaryBuilderCase { return Create>(); } Status Visit(const FixedSizeBinaryType&) { return CreateFor(); } + Status Visit(const Decimal16Type&) { return CreateFor(); } + Status Visit(const Decimal32Type&) { return CreateFor(); } + Status Visit(const Decimal64Type&) { return CreateFor(); } Status Visit(const Decimal128Type&) { return CreateFor(); } Status Visit(const Decimal256Type&) { return CreateFor(); } @@ -138,6 +141,9 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, BUILDER_CASE(LargeString); BUILDER_CASE(LargeBinary); BUILDER_CASE(FixedSizeBinary); + BUILDER_CASE(Decimal16); + BUILDER_CASE(Decimal32); + BUILDER_CASE(Decimal64); BUILDER_CASE(Decimal128); BUILDER_CASE(Decimal256); diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc index 4c6f97faf95..1e5c350e63c 100644 --- a/cpp/src/arrow/compare.cc +++ b/cpp/src/arrow/compare.cc @@ -264,6 +264,11 @@ class RangeDataEqualsImpl { Status Visit(const LargeListType& type) { return CompareList(type); } + template + Status Visit(const BaseDecimalArray& left) { + return Visit(checked_cast(left)); + } + Status Visit(const FixedSizeListType& type) { const auto list_size = type.list_size(); const ArrayData& left_data = *left_.child_data[0]; @@ -605,14 +610,9 @@ class TypeEqualsVisitor { return Status::OK(); } - Status Visit(const Decimal128Type& left) { - const auto& right = checked_cast(right_); - result_ = left.precision() == right.precision() && left.scale() == right.scale(); - return Status::OK(); - } - - Status Visit(const Decimal256Type& left) { - const auto& right = checked_cast(right_); + template + Status Visit(const BaseDecimalType& left) { + const auto& right = checked_cast&>(right_); result_ = left.precision() == right.precision() && left.scale() == right.scale(); return Status::OK(); } @@ -721,14 +721,9 @@ class ScalarEqualsVisitor { return Status::OK(); } - Status Visit(const Decimal128Scalar& left) { - const auto& right = checked_cast(right_); - result_ = left.value == right.value; - return Status::OK(); - } - - Status Visit(const Decimal256Scalar& left) { - const auto& right = checked_cast(right_); + template + Status Visit(const BaseDecimalScalar& left) { + const auto& right = checked_cast&>(right_); result_ = left.value == right.value; return Status::OK(); } diff --git a/cpp/src/arrow/ipc/json_simple.cc b/cpp/src/arrow/ipc/json_simple.cc index fba8194aeb1..1a7a142f9f8 100644 --- a/cpp/src/arrow/ipc/json_simple.cc +++ b/cpp/src/arrow/ipc/json_simple.cc @@ -332,10 +332,19 @@ class DecimalConverter final const DecimalSubtype* decimal_type_; }; -template ::BuilderType> -using Decimal128Converter = DecimalConverter; -template ::BuilderType> -using Decimal256Converter = DecimalConverter; +#define DECL_DECIMAL_CONVERTER(width) \ + template ::BuilderType> \ + using Decimal##width##Converter = \ + DecimalConverter; + +DECL_DECIMAL_CONVERTER(16) +DECL_DECIMAL_CONVERTER(32) +DECL_DECIMAL_CONVERTER(64) +DECL_DECIMAL_CONVERTER(128) +DECL_DECIMAL_CONVERTER(256) + +#undef DECL_DECIMAL_CONVERTER // ------------------------------------------------------------------------ // Converter for timestamp arrays @@ -786,6 +795,9 @@ Status GetDictConverter(const std::shared_ptr& type, PARAM_CONVERTER_CASE(Type::LARGE_BINARY, StringConverter, LargeBinaryType) SIMPLE_CONVERTER_CASE(Type::FIXED_SIZE_BINARY, FixedSizeBinaryConverter, FixedSizeBinaryType) + SIMPLE_CONVERTER_CASE(Type::DECIMAL16, Decimal16Converter, Decimal16Type) + SIMPLE_CONVERTER_CASE(Type::DECIMAL32, Decimal32Converter, Decimal32Type) + SIMPLE_CONVERTER_CASE(Type::DECIMAL64, Decimal64Converter, Decimal64Type) SIMPLE_CONVERTER_CASE(Type::DECIMAL128, Decimal128Converter, Decimal128Type) SIMPLE_CONVERTER_CASE(Type::DECIMAL256, Decimal256Converter, Decimal256Type) default: @@ -843,6 +855,9 @@ Status GetConverter(const std::shared_ptr& type, SIMPLE_CONVERTER_CASE(Type::LARGE_STRING, StringConverter) SIMPLE_CONVERTER_CASE(Type::LARGE_BINARY, StringConverter) SIMPLE_CONVERTER_CASE(Type::FIXED_SIZE_BINARY, FixedSizeBinaryConverter<>) + SIMPLE_CONVERTER_CASE(Type::DECIMAL16, Decimal16Converter<>) + SIMPLE_CONVERTER_CASE(Type::DECIMAL32, Decimal32Converter<>) + SIMPLE_CONVERTER_CASE(Type::DECIMAL64, Decimal64Converter<>) SIMPLE_CONVERTER_CASE(Type::DECIMAL128, Decimal128Converter<>) SIMPLE_CONVERTER_CASE(Type::DECIMAL256, Decimal256Converter<>) SIMPLE_CONVERTER_CASE(Type::SPARSE_UNION, UnionConverter) diff --git a/cpp/src/arrow/ipc/json_simple_test.cc b/cpp/src/arrow/ipc/json_simple_test.cc index c5358ac89f1..b44a17ce09d 100644 --- a/cpp/src/arrow/ipc/json_simple_test.cc +++ b/cpp/src/arrow/ipc/json_simple_test.cc @@ -41,6 +41,7 @@ #include "arrow/util/bitmap_builders.h" #include "arrow/util/checked_cast.h" #include "arrow/util/decimal.h" +#include "arrow/util/decimal_type_traits.h" #if defined(_MSC_VER) // "warning C4307: '+': integral constant overflow" @@ -498,8 +499,21 @@ TEST(TestFixedSizeBinary, Dictionary) { ASSERT_RAISES(Invalid, ArrayFromJSON(dictionary(int8(), type), R"(["x"])", &array)); } -template -void TestDecimalBasic(std::shared_ptr type) { +template +class TestDecimal : public testing::Test {}; +using DecimalTypes = + ::testing::Types, DecimalTypeTraits<32>, DecimalTypeTraits<64>, + DecimalTypeTraits<128>, DecimalTypeTraits<256>>; + +TYPED_TEST_SUITE(TestDecimal, DecimalTypes); + +TYPED_TEST(TestDecimal, Basic) { + using TypeClass = typename TypeParam::TypeClass; + using DecimalBuilder = typename TypeParam::BuilderType; + using DecimalValue = typename TypeParam::ValueType; + + auto type = std::make_shared(5, 4); + std::shared_ptr expected, actual; ASSERT_OK(ArrayFromJSON(type, "[]", &actual)); @@ -510,54 +524,46 @@ void TestDecimalBasic(std::shared_ptr type) { } AssertArraysEqual(*expected, *actual); - ASSERT_OK(ArrayFromJSON(type, "[\"123.4567\", \"-78.9000\"]", &actual)); + ASSERT_OK(ArrayFromJSON(type, "[\"1.2345\", \"-3.2000\"]", &actual)); ASSERT_OK(actual->ValidateFull()); { DecimalBuilder builder(type); - ASSERT_OK(builder.Append(DecimalValue(1234567))); - ASSERT_OK(builder.Append(DecimalValue(-789000))); + ASSERT_OK(builder.Append(DecimalValue(12345))); + ASSERT_OK(builder.Append(DecimalValue(-32000))); ASSERT_OK(builder.Finish(&expected)); } AssertArraysEqual(*expected, *actual); - ASSERT_OK(ArrayFromJSON(type, "[\"123.4567\", null]", &actual)); + ASSERT_OK(ArrayFromJSON(type, "[\"1.2345\", null]", &actual)); ASSERT_OK(actual->ValidateFull()); { DecimalBuilder builder(type); - ASSERT_OK(builder.Append(DecimalValue(1234567))); + ASSERT_OK(builder.Append(DecimalValue(12345))); ASSERT_OK(builder.AppendNull()); ASSERT_OK(builder.Finish(&expected)); } AssertArraysEqual(*expected, *actual); } -TEST(TestDecimal128, Basics) { - TestDecimalBasic(decimal128(10, 4)); -} - -TEST(TestDecimal256, Basics) { - TestDecimalBasic(decimal256(10, 4)); -} - -TEST(TestDecimal, Errors) { - for (std::shared_ptr type : {decimal128(10, 4), decimal256(10, 4)}) { - std::shared_ptr array; +TYPED_TEST(TestDecimal, Errors) { + using TypeClass = typename TypeParam::TypeClass; + auto type = std::make_shared(5, 4); - ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[0]", &array)); - ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[12.3456]", &array)); - // Bad scale - ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.345\"]", &array)); - ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.34560\"]", &array)); - } -} + std::shared_ptr array; -TEST(TestDecimal, Dictionary) { - for (std::shared_ptr type : {decimal128(10, 2), decimal256(10, 2)}) { - AssertJSONDictArray(int32(), type, - R"(["123.45", "-78.90", "-78.90", null, "123.45"])", - /*indices=*/"[0, 1, 1, null, 0]", - /*values=*/R"(["123.45", "-78.90"])"); - } + ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[0]", &array)); + ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[1.2345]", &array)); + // Bad scale + ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.345\"]", &array)); + ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.34560\"]", &array)); +} + +TYPED_TEST(TestDecimal, Dictionary) { + using TypeClass = typename TypeParam::TypeClass; + auto type = std::make_shared(5, 2); + AssertJSONDictArray(int32(), type, R"(["123.45", "-78.90", "-78.90", null, "123.45"])", + /*indices=*/"[0, 1, 1, null, 0]", + /*values=*/R"(["123.45", "-78.90"])"); } TEST(TestList, IntegerList) { diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc index 8c2ac376d1e..9bf37328151 100644 --- a/cpp/src/arrow/pretty_print.cc +++ b/cpp/src/arrow/pretty_print.cc @@ -222,12 +222,8 @@ class ArrayPrinter : public PrettyPrinter { return Status::OK(); } - Status WriteDataValues(const Decimal128Array& array) { - WriteValues(array, [&](int64_t i) { (*sink_) << array.FormatValue(i); }); - return Status::OK(); - } - - Status WriteDataValues(const Decimal256Array& array) { + template + Status WriteDataValues(const BaseDecimalArray& array) { WriteValues(array, [&](int64_t i) { (*sink_) << array.FormatValue(i); }); return Status::OK(); } diff --git a/cpp/src/arrow/pretty_print_test.cc b/cpp/src/arrow/pretty_print_test.cc index 538e7365185..078bd5abccd 100644 --- a/cpp/src/arrow/pretty_print_test.cc +++ b/cpp/src/arrow/pretty_print_test.cc @@ -499,13 +499,14 @@ TEST_F(TestPrettyPrint, FixedSizeBinaryType) { } TEST_F(TestPrettyPrint, DecimalTypes) { - int32_t p = 19; + int32_t p = 5; int32_t s = 4; - for (auto type : {decimal128(p, s), decimal256(p, s)}) { - auto array = ArrayFromJSON(type, "[\"123.4567\", \"456.7891\", null]"); + for (auto type : {decimal16(p, s), decimal32(p, s), decimal64(p, s), decimal128(p, s), + decimal256(p, s)}) { + auto array = ArrayFromJSON(type, "[\"1.4567\", \"3.2765\", null]"); - static const char* ex = "[\n 123.4567,\n 456.7891,\n null\n]"; + static const char* ex = "[\n 1.4567,\n 3.2765,\n null\n]"; CheckArray(*array, {0}, ex); } } diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc index 09245285030..27903edc00c 100644 --- a/cpp/src/arrow/python/arrow_to_pandas.cc +++ b/cpp/src/arrow/python/arrow_to_pandas.cc @@ -167,6 +167,9 @@ static inline bool ListTypeSupported(const DataType& type) { case Type::UINT64: case Type::FLOAT: case Type::DOUBLE: + case Type::DECIMAL16: + case Type::DECIMAL32: + case Type::DECIMAL64: case Type::DECIMAL128: case Type::DECIMAL256: case Type::BINARY: @@ -1021,7 +1024,9 @@ struct ObjectWriterVisitor { } template - enable_if_t::value || is_fixed_size_binary_type::value, + enable_if_t<(is_base_binary_type::value || + is_fixed_size_binary_type::value) && + !is_decimal_type::value, Status> Visit(const Type& type) { auto WrapValue = [](const util::string_view& view, PyObject** out) { @@ -1094,7 +1099,8 @@ struct ObjectWriterVisitor { return Status::OK(); } - Status Visit(const Decimal128Type& type) { + template + Status Visit(const BaseDecimalType& type) { OwnedRef decimal; OwnedRef Decimal; RETURN_NOT_OK(internal::ImportModule("decimal", &decimal)); @@ -1102,32 +1108,8 @@ struct ObjectWriterVisitor { PyObject* decimal_constructor = Decimal.obj(); for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = checked_cast(*data.chunk(c)); - - for (int64_t i = 0; i < arr.length(); ++i) { - if (arr.IsNull(i)) { - Py_INCREF(Py_None); - *out_values++ = Py_None; - } else { - *out_values++ = - internal::DecimalFromString(decimal_constructor, arr.FormatValue(i)); - RETURN_IF_PYERROR(); - } - } - } - - return Status::OK(); - } - - Status Visit(const Decimal256Type& type) { - OwnedRef decimal; - OwnedRef Decimal; - RETURN_NOT_OK(internal::ImportModule("decimal", &decimal)); - RETURN_NOT_OK(internal::ImportFromModule(decimal.obj(), "Decimal", &Decimal)); - PyObject* decimal_constructor = Decimal.obj(); - - for (int c = 0; c < data.num_chunks(); c++) { - const auto& arr = checked_cast(*data.chunk(c)); + const auto& arr = + checked_cast&>(*data.chunk(c)); for (int64_t i = 0; i < arr.length(); ++i) { if (arr.IsNull(i)) { @@ -1871,6 +1853,9 @@ static Status GetPandasWriterType(const ChunkedArray& data, const PandasOptions& case Type::STRUCT: // fall through case Type::TIME32: // fall through case Type::TIME64: // fall through + case Type::DECIMAL16: // fall through + case Type::DECIMAL32: // fall through + case Type::DECIMAL64: // fall through case Type::DECIMAL128: // fall through case Type::DECIMAL256: // fall through *output_type = PandasWriter::OBJECT; diff --git a/cpp/src/arrow/python/decimal.cc b/cpp/src/arrow/python/decimal.cc index 67389095b94..8f36812da26 100644 --- a/cpp/src/arrow/python/decimal.cc +++ b/cpp/src/arrow/python/decimal.cc @@ -166,6 +166,33 @@ Status InternalDecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, } // namespace +Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type, + Decimal16* out) { + return InternalDecimalFromPythonDecimal(python_decimal, arrow_type, out); +} + +Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, Decimal16* out) { + return InternalDecimalFromPyObject(obj, arrow_type, out); +} + +Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type, + Decimal32* out) { + return InternalDecimalFromPythonDecimal(python_decimal, arrow_type, out); +} + +Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, Decimal32* out) { + return InternalDecimalFromPyObject(obj, arrow_type, out); +} + +Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type, + Decimal64* out) { + return InternalDecimalFromPythonDecimal(python_decimal, arrow_type, out); +} + +Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, Decimal64* out) { + return InternalDecimalFromPyObject(obj, arrow_type, out); +} + Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type, Decimal128* out) { return InternalDecimalFromPythonDecimal(python_decimal, arrow_type, out); diff --git a/cpp/src/arrow/python/decimal.h b/cpp/src/arrow/python/decimal.h index 1187037aed2..5a7698a3929 100644 --- a/cpp/src/arrow/python/decimal.h +++ b/cpp/src/arrow/python/decimal.h @@ -24,6 +24,12 @@ namespace arrow { +template +class DecimalAnyWidth; + +using Decimal16 = DecimalAnyWidth<16>; +using Decimal32 = DecimalAnyWidth<32>; +using Decimal64 = DecimalAnyWidth<64>; class Decimal128; class Decimal256; @@ -56,6 +62,57 @@ ARROW_PYTHON_EXPORT PyObject* DecimalFromString(PyObject* decimal_constructor, const std::string& decimal_string); +// \brief Convert a Python decimal to an Arrow Decimal16 object +// \param[in] python_decimal A Python decimal.Decimal instance +// \param[in] arrow_type An instance of arrow::DecimalType +// \param[out] out A pointer to a Decimal16 +// \return The status of the operation +ARROW_PYTHON_EXPORT +Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type, + Decimal16* out); + +// \brief Convert a Python object to an Arrow Decimal16 object +// \param[in] python_decimal A Python int or decimal.Decimal instance +// \param[in] arrow_type An instance of arrow::DecimalType +// \param[out] out A pointer to a Decimal16 +// \return The status of the operation +ARROW_PYTHON_EXPORT +Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, Decimal16* out); + +// \brief Convert a Python decimal to an Arrow Decimal32 object +// \param[in] python_decimal A Python decimal.Decimal instance +// \param[in] arrow_type An instance of arrow::DecimalType +// \param[out] out A pointer to a Decimal32 +// \return The status of the operation +ARROW_PYTHON_EXPORT +Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type, + Decimal32* out); + +// \brief Convert a Python object to an Arrow Decimal32 object +// \param[in] python_decimal A Python int or decimal.Decimal instance +// \param[in] arrow_type An instance of arrow::DecimalType +// \param[out] out A pointer to a Decimal32 +// \return The status of the operation +ARROW_PYTHON_EXPORT +Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, Decimal32* out); + +// \brief Convert a Python decimal to an Arrow Decimal64 object +// \param[in] python_decimal A Python decimal.Decimal instance +// \param[in] arrow_type An instance of arrow::DecimalType +// \param[out] out A pointer to a Decimal64 +// \return The status of the operation +ARROW_PYTHON_EXPORT +Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type, + Decimal64* out); + +// \brief Convert a Python object to an Arrow Decimal64 object +// \param[in] python_decimal A Python int or decimal.Decimal instance +// \param[in] arrow_type An instance of arrow::DecimalType +// \param[out] out A pointer to a Decimal64 +// \return The status of the operation +ARROW_PYTHON_EXPORT +Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, Decimal64* out); + // \brief Convert a Python decimal to an Arrow Decimal128 object // \param[in] python_decimal A Python decimal.Decimal instance // \param[in] arrow_type An instance of arrow::DecimalType diff --git a/cpp/src/arrow/python/python_test.cc b/cpp/src/arrow/python/python_test.cc index 33e0ee9b1c9..799b7c1d409 100644 --- a/cpp/src/arrow/python/python_test.cc +++ b/cpp/src/arrow/python/python_test.cc @@ -28,6 +28,7 @@ #include "arrow/table.h" #include "arrow/testing/gtest_util.h" #include "arrow/util/decimal.h" +#include "arrow/util/decimal_type_traits.h" #include "arrow/util/optional.h" #include "arrow/python/arrow_to_pandas.h" @@ -358,6 +359,12 @@ void DecimalTestFromPythonDecimalRescale(std::shared_ptr type, TEST_F(DecimalTest, FromPythonDecimalRescaleNotTruncateable) { // We fail when truncating values that would lose data if cast to a decimal type with // lower scale + DecimalTestFromPythonDecimalRescale(::arrow::decimal16(5, 2), + this->CreatePythonDecimal("1.001"), {}); + DecimalTestFromPythonDecimalRescale(::arrow::decimal32(10, 2), + this->CreatePythonDecimal("1.001"), {}); + DecimalTestFromPythonDecimalRescale(::arrow::decimal64(10, 2), + this->CreatePythonDecimal("1.001"), {}); DecimalTestFromPythonDecimalRescale(::arrow::decimal128(10, 2), this->CreatePythonDecimal("1.001"), {}); DecimalTestFromPythonDecimalRescale(::arrow::decimal256(10, 2), @@ -367,6 +374,12 @@ TEST_F(DecimalTest, FromPythonDecimalRescaleNotTruncateable) { TEST_F(DecimalTest, FromPythonDecimalRescaleTruncateable) { // We allow truncation of values that do not lose precision when dividing by 10 * the // difference between the scales, e.g., 1.000 -> 1.00 + DecimalTestFromPythonDecimalRescale(::arrow::decimal16(5, 2), + this->CreatePythonDecimal("1.000"), 100); + DecimalTestFromPythonDecimalRescale(::arrow::decimal32(10, 2), + this->CreatePythonDecimal("1.000"), 100); + DecimalTestFromPythonDecimalRescale(::arrow::decimal64(10, 2), + this->CreatePythonDecimal("1.000"), 100); DecimalTestFromPythonDecimalRescale( ::arrow::decimal128(10, 2), this->CreatePythonDecimal("1.000"), 100); DecimalTestFromPythonDecimalRescale( @@ -374,25 +387,33 @@ TEST_F(DecimalTest, FromPythonDecimalRescaleTruncateable) { } TEST_F(DecimalTest, FromPythonNegativeDecimalRescale) { + DecimalTestFromPythonDecimalRescale( + ::arrow::decimal16(5, 4), this->CreatePythonDecimal("-1.000"), -10000); + DecimalTestFromPythonDecimalRescale( + ::arrow::decimal32(10, 9), this->CreatePythonDecimal("-1.000"), -1000000000); + DecimalTestFromPythonDecimalRescale( + ::arrow::decimal64(10, 9), this->CreatePythonDecimal("-1.000"), -1000000000); DecimalTestFromPythonDecimalRescale( ::arrow::decimal128(10, 9), this->CreatePythonDecimal("-1.000"), -1000000000); DecimalTestFromPythonDecimalRescale( ::arrow::decimal256(10, 9), this->CreatePythonDecimal("-1.000"), -1000000000); } -TEST_F(DecimalTest, Decimal128FromPythonInteger) { - Decimal128 value; - OwnedRef python_long(PyLong_FromLong(42)); - auto type = ::arrow::decimal128(10, 2); - const auto& decimal_type = checked_cast(*type); - ASSERT_OK(internal::DecimalFromPyObject(python_long.obj(), decimal_type, &value)); - ASSERT_EQ(4200, value); -} +template +class DecimalTestConversion : public testing::Test {}; +using DecimalTypes = + ::testing::Types, DecimalTypeTraits<32>, DecimalTypeTraits<64>, + DecimalTypeTraits<128>, DecimalTypeTraits<256>>; -TEST_F(DecimalTest, Decimal256FromPythonInteger) { - Decimal256 value; +TYPED_TEST_SUITE(DecimalTestConversion, DecimalTypes); + +TYPED_TEST(DecimalTestConversion, Basics) { + using TypeClass = typename TypeParam::TypeClass; + using ValueType = typename TypeParam::ValueType; + + ValueType value; OwnedRef python_long(PyLong_FromLong(42)); - auto type = ::arrow::decimal256(10, 2); + auto type = std::make_shared(5, 2); const auto& decimal_type = checked_cast(*type); ASSERT_OK(internal::DecimalFromPyObject(python_long.obj(), decimal_type, &value)); ASSERT_EQ(4200, value); diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc index b136bec9709..2d9de621af2 100644 --- a/cpp/src/arrow/python/python_to_arrow.cc +++ b/cpp/src/arrow/python/python_to_arrow.cc @@ -164,6 +164,24 @@ class PyValue { return value; } + static Result Convert(const Decimal16Type* type, const O&, I obj) { + Decimal16 value; + RETURN_NOT_OK(internal::DecimalFromPyObject(obj, *type, &value)); + return value; + } + + static Result Convert(const Decimal32Type* type, const O&, I obj) { + Decimal32 value; + RETURN_NOT_OK(internal::DecimalFromPyObject(obj, *type, &value)); + return value; + } + + static Result Convert(const Decimal64Type* type, const O&, I obj) { + Decimal64 value; + RETURN_NOT_OK(internal::DecimalFromPyObject(obj, *type, &value)); + return value; + } + static Result Convert(const Decimal128Type* type, const O&, I obj) { Decimal128 value; RETURN_NOT_OK(internal::DecimalFromPyObject(obj, *type, &value)); diff --git a/cpp/src/arrow/scalar.cc b/cpp/src/arrow/scalar.cc index eca711d7c4f..4c01c38adca 100644 --- a/cpp/src/arrow/scalar.cc +++ b/cpp/src/arrow/scalar.cc @@ -69,6 +69,11 @@ struct ScalarHashImpl { return StdHash(s.value.days) & StdHash(s.value.days); } + template ::type> + Status Visit(const BaseDecimalScalar& s) { + return StdHash(s.value.Value()); + } + Status Visit(const Decimal128Scalar& s) { return StdHash(s.value.low_bits()) & StdHash(s.value.high_bits()); } diff --git a/cpp/src/arrow/scalar.h b/cpp/src/arrow/scalar.h index 2888874d292..01decf9637c 100644 --- a/cpp/src/arrow/scalar.h +++ b/cpp/src/arrow/scalar.h @@ -36,6 +36,7 @@ #include "arrow/type_traits.h" #include "arrow/util/compare.h" #include "arrow/util/decimal.h" +#include "arrow/util/decimal_type_traits.h" #include "arrow/util/string_view.h" #include "arrow/util/visibility.h" @@ -339,26 +340,16 @@ struct ARROW_EXPORT DurationScalar : public TemporalScalar { using TemporalScalar::TemporalScalar; }; -struct ARROW_EXPORT Decimal128Scalar : public Scalar { +template +struct BaseDecimalScalar : public Scalar { using Scalar::Scalar; - using TypeClass = Decimal128Type; - using ValueType = Decimal128; + using TypeClass = typename DecimalTypeTraits::TypeClass; + using ValueType = typename DecimalTypeTraits::ValueType; - Decimal128Scalar(Decimal128 value, std::shared_ptr type) + BaseDecimalScalar(ValueType value, std::shared_ptr type) : Scalar(std::move(type), true), value(value) {} - Decimal128 value; -}; - -struct ARROW_EXPORT Decimal256Scalar : public Scalar { - using Scalar::Scalar; - using TypeClass = Decimal256Type; - using ValueType = Decimal256; - - Decimal256Scalar(Decimal256 value, std::shared_ptr type) - : Scalar(std::move(type), true), value(value) {} - - Decimal256 value; + ValueType value; }; struct ARROW_EXPORT BaseListScalar : public Scalar { diff --git a/cpp/src/arrow/scalar_test.cc b/cpp/src/arrow/scalar_test.cc index 30a39e6e4c0..64c2a5aa234 100644 --- a/cpp/src/arrow/scalar_test.cc +++ b/cpp/src/arrow/scalar_test.cc @@ -31,6 +31,7 @@ #include "arrow/status.h" #include "arrow/testing/gtest_util.h" #include "arrow/type_traits.h" +#include "arrow/util/decimal_type_traits.h" namespace arrow { @@ -328,29 +329,24 @@ TYPED_TEST(TestRealScalar, StructOf) { this->TestStructOf(); } TYPED_TEST(TestRealScalar, ListOf) { this->TestListOf(); } -TEST(TestDecimal128Scalar, Basics) { - auto ty = decimal128(3, 2); - auto pi = Decimal128Scalar(Decimal128("3.14"), ty); - auto null = MakeNullScalar(ty); +template +class TestDecimalScalar : public testing::Test {}; +using DecimalTypes = + ::testing::Types, DecimalTypeTraits<32>, DecimalTypeTraits<64>, + DecimalTypeTraits<128>, DecimalTypeTraits<256>>; - ASSERT_EQ(pi.value, Decimal128("3.14")); +TYPED_TEST_SUITE(TestDecimalScalar, DecimalTypes); - // test Array.GetScalar - auto arr = ArrayFromJSON(ty, "[null, \"3.14\"]"); - ASSERT_OK_AND_ASSIGN(auto first, arr->GetScalar(0)); - ASSERT_OK_AND_ASSIGN(auto second, arr->GetScalar(1)); - ASSERT_TRUE(first->Equals(null)); - ASSERT_FALSE(first->Equals(pi)); - ASSERT_TRUE(second->Equals(pi)); - ASSERT_FALSE(second->Equals(null)); -} +TYPED_TEST(TestDecimalScalar, Basics) { + using ScalarType = typename TypeParam::ScalarType; + using TypeClass = typename TypeParam::TypeClass; + using ValueType = typename TypeParam::ValueType; -TEST(TestDecimal256Scalar, Basics) { - auto ty = decimal256(3, 2); - auto pi = Decimal256Scalar(Decimal256("3.14"), ty); + auto ty = std::make_shared(3, 2); + auto pi = ScalarType(ValueType("3.14"), ty); auto null = MakeNullScalar(ty); - ASSERT_EQ(pi.value, Decimal256("3.14")); + ASSERT_EQ(pi.value, ValueType("3.14")); // test Array.GetScalar auto arr = ArrayFromJSON(ty, "[null, \"3.14\"]"); diff --git a/cpp/src/arrow/testing/gtest_util.cc b/cpp/src/arrow/testing/gtest_util.cc index aa9d22dae2f..9307cc771ff 100644 --- a/cpp/src/arrow/testing/gtest_util.cc +++ b/cpp/src/arrow/testing/gtest_util.cc @@ -69,6 +69,9 @@ std::vector AllTypeIds() { Type::HALF_FLOAT, Type::FLOAT, Type::DOUBLE, + Type::DECIMAL16, + Type::DECIMAL32, + Type::DECIMAL64, Type::DECIMAL128, Type::DECIMAL256, Type::DATE32, diff --git a/cpp/src/arrow/testing/json_internal.cc b/cpp/src/arrow/testing/json_internal.cc index 21f7514289d..4d5e4c7e5ef 100644 --- a/cpp/src/arrow/testing/json_internal.cc +++ b/cpp/src/arrow/testing/json_internal.cc @@ -298,14 +298,8 @@ class SchemaWriter { writer_->Int(type.list_size()); } - void WriteTypeMetadata(const Decimal128Type& type) { - writer_->Key("precision"); - writer_->Int(type.precision()); - writer_->Key("scale"); - writer_->Int(type.scale()); - } - - void WriteTypeMetadata(const Decimal256Type& type) { + template + void WriteTypeMetadata(const BaseDecimalType& type) { writer_->Key("precision"); writer_->Int(type.precision()); writer_->Key("scale"); @@ -384,6 +378,9 @@ class SchemaWriter { return WritePrimitive("fixedsizebinary", type); } + Status Visit(const Decimal16Type& type) { return WritePrimitive("decimal16", type); } + Status Visit(const Decimal32Type& type) { return WritePrimitive("decimal32", type); } + Status Visit(const Decimal64Type& type) { return WritePrimitive("decimal64", type); } Status Visit(const Decimal128Type& type) { return WritePrimitive("decimal", type); } Status Visit(const Decimal256Type& type) { return WritePrimitive("decimal256", type); } Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); } @@ -549,23 +546,12 @@ class ArrayWriter { } } - void WriteDataValues(const Decimal128Array& arr) { - static const char null_string[] = "0"; - for (int64_t i = 0; i < arr.length(); ++i) { - if (arr.IsValid(i)) { - const Decimal128 value(arr.GetValue(i)); - writer_->String(value.ToIntegerString()); - } else { - writer_->String(null_string, sizeof(null_string)); - } - } - } - - void WriteDataValues(const Decimal256Array& arr) { + template + void WriteDataValues(const BaseDecimalArray& arr) { static const char null_string[] = "0"; for (int64_t i = 0; i < arr.length(); ++i) { if (arr.IsValid(i)) { - const Decimal256 value(arr.GetValue(i)); + const typename BaseDecimalArray::ValueType value(arr.GetValue(i)); writer_->String(value.ToIntegerString()); } else { writer_->String(null_string, sizeof(null_string)); @@ -860,14 +846,27 @@ Status GetDecimal(const RjObject& json_type, std::shared_ptr* type) { bit_width = maybe_bit_width.ValueOrDie(); } - if (bit_width == 128) { - *type = decimal128(precision, scale); - } else if (bit_width == 256) { - *type = decimal256(precision, scale); - } else { - return Status::Invalid("Only 128 bit and 256 Decimals are supported. Received", - bit_width); + switch (bit_width) { + case 16: + *type = decimal16(precision, scale); + break; + case 32: + *type = decimal32(precision, scale); + break; + case 64: + *type = decimal64(precision, scale); + break; + case 128: + *type = decimal128(precision, scale); + break; + case 256: + *type = decimal256(precision, scale); + break; + default: + return Status::Invalid("Only 128 bit and 256 Decimals are supported. Received", + bit_width); } + return Status::OK(); } diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index 12d3951865f..20d71818a68 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -35,6 +35,7 @@ #include "arrow/result.h" #include "arrow/status.h" #include "arrow/util/checked_cast.h" +#include "arrow/util/decimal_type_traits.h" #include "arrow/util/hash_util.h" #include "arrow/util/hashing.h" #include "arrow/util/key_value_metadata.h" @@ -66,6 +67,12 @@ constexpr Type::type FixedSizeBinaryType::type_id; constexpr Type::type StructType::type_id; +constexpr Type::type Decimal16Type::type_id; + +constexpr Type::type Decimal32Type::type_id; + +constexpr Type::type Decimal64Type::type_id; + constexpr Type::type Decimal128Type::type_id; constexpr Type::type Decimal256Type::type_id; @@ -130,6 +137,9 @@ std::string ToString(Type::type id) { TO_STRING_CASE(HALF_FLOAT) TO_STRING_CASE(FLOAT) TO_STRING_CASE(DOUBLE) + TO_STRING_CASE(DECIMAL16) + TO_STRING_CASE(DECIMAL32) + TO_STRING_CASE(DECIMAL64) TO_STRING_CASE(DECIMAL128) TO_STRING_CASE(DECIMAL256) TO_STRING_CASE(DATE32) @@ -794,35 +804,22 @@ int32_t DecimalType::DecimalSize(int32_t precision) { } // ---------------------------------------------------------------------- -// Decimal128 type +// Decimal type -Decimal128Type::Decimal128Type(int32_t precision, int32_t scale) - : DecimalType(type_id, 16, precision, scale) { +template +BaseDecimalType::BaseDecimalType(int32_t precision, int32_t scale) + : DecimalType(DecimalTypeTraits::Id, (width >> 3), precision, scale) { ARROW_CHECK_GE(precision, kMinPrecision); ARROW_CHECK_LE(precision, kMaxPrecision); } -Result> Decimal128Type::Make(int32_t precision, int32_t scale) { +template +Result> BaseDecimalType::Make(int32_t precision, + int32_t scale) { if (precision < kMinPrecision || precision > kMaxPrecision) { return Status::Invalid("Decimal precision out of range: ", precision); } - return std::make_shared(precision, scale); -} - -// ---------------------------------------------------------------------- -// Decimal256 type - -Decimal256Type::Decimal256Type(int32_t precision, int32_t scale) - : DecimalType(type_id, 32, precision, scale) { - ARROW_CHECK_GE(precision, kMinPrecision); - ARROW_CHECK_LE(precision, kMaxPrecision); -} - -Result> Decimal256Type::Make(int32_t precision, int32_t scale) { - if (precision < kMinPrecision || precision > kMaxPrecision) { - return Status::Invalid("Decimal precision out of range: ", precision); - } - return std::make_shared(precision, scale); + return std::make_shared::TypeClass>(precision, scale); } // ---------------------------------------------------------------------- @@ -2195,6 +2192,18 @@ std::shared_ptr decimal(int32_t precision, int32_t scale) { : decimal256(precision, scale); } +std::shared_ptr decimal16(int32_t precision, int32_t scale) { + return std::make_shared(precision, scale); +} + +std::shared_ptr decimal32(int32_t precision, int32_t scale) { + return std::make_shared(precision, scale); +} + +std::shared_ptr decimal64(int32_t precision, int32_t scale) { + return std::make_shared(precision, scale); +} + std::shared_ptr decimal128(int32_t precision, int32_t scale) { return std::make_shared(precision, scale); } @@ -2203,16 +2212,17 @@ std::shared_ptr decimal256(int32_t precision, int32_t scale) { return std::make_shared(precision, scale); } -std::string Decimal128Type::ToString() const { +template +std::string BaseDecimalType::ToString() const { std::stringstream s; - s << "decimal(" << precision_ << ", " << scale_ << ")"; + s << type_name() << "(" << precision_ << ", " << scale_ << ")"; return s.str(); } -std::string Decimal256Type::ToString() const { - std::stringstream s; - s << "decimal256(" << precision_ << ", " << scale_ << ")"; - return s.str(); -} +template class BaseDecimalType<16>; +template class BaseDecimalType<32>; +template class BaseDecimalType<64>; +template class BaseDecimalType<128>; +template class BaseDecimalType<256>; } // namespace arrow diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index 127ed598399..edfcde60bc2 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -30,6 +30,7 @@ #include "arrow/result.h" #include "arrow/type_fwd.h" // IWYU pragma: export #include "arrow/util/checked_cast.h" +#include "arrow/util/decimal_meta.h" #include "arrow/util/macros.h" #include "arrow/util/variant.h" #include "arrow/util/visibility.h" @@ -892,46 +893,59 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType { int32_t scale_; }; -/// \brief Concrete type class for 128-bit decimal data -class ARROW_EXPORT Decimal128Type : public DecimalType { +/// \brief Template type class for decimal data +template +class ARROW_EXPORT BaseDecimalType : public DecimalType { public: - static constexpr Type::type type_id = Type::DECIMAL128; - - static constexpr const char* type_name() { return "decimal"; } + static constexpr const char* type_name() { return DecimalMeta::name; } - /// Decimal128Type constructor that aborts on invalid input. - explicit Decimal128Type(int32_t precision, int32_t scale); + /// BaseDecimalType constructor that aborts on invalid input. + explicit BaseDecimalType(int32_t precision, int32_t scale); - /// Decimal128Type constructor that returns an error on invalid input. + /// BaseDecimalType constructor that returns an error on invalid input. static Result> Make(int32_t precision, int32_t scale); std::string ToString() const override; - std::string name() const override { return "decimal"; } + std::string name() const override { return DecimalMeta::name; } static constexpr int32_t kMinPrecision = 1; - static constexpr int32_t kMaxPrecision = 38; - static constexpr int32_t kByteWidth = 16; + static constexpr int32_t kMaxPrecision = DecimalMeta::max_precision; + static constexpr int32_t kByteWidth = width / 8; }; -/// \brief Concrete type class for 256-bit decimal data -class ARROW_EXPORT Decimal256Type : public DecimalType { +/// \brief Concrete type class for decimal 16-bit data +class ARROW_EXPORT Decimal16Type : public BaseDecimalType<16> { public: - static constexpr Type::type type_id = Type::DECIMAL256; - - static constexpr const char* type_name() { return "decimal256"; } + static constexpr Type::type type_id = Type::DECIMAL16; + using BaseDecimalType<16>::BaseDecimalType; +}; - /// Decimal256Type constructor that aborts on invalid input. - explicit Decimal256Type(int32_t precision, int32_t scale); +/// \brief Concrete type class for decimal 32-bit data +class ARROW_EXPORT Decimal32Type : public BaseDecimalType<32> { + public: + static constexpr Type::type type_id = Type::DECIMAL32; + using BaseDecimalType<32>::BaseDecimalType; +}; - /// Decimal256Type constructor that returns an error on invalid input. - static Result> Make(int32_t precision, int32_t scale); +/// \brief Concrete type class for decimal 64-bit data +class ARROW_EXPORT Decimal64Type : public BaseDecimalType<64> { + public: + static constexpr Type::type type_id = Type::DECIMAL64; + using BaseDecimalType<64>::BaseDecimalType; +}; - std::string ToString() const override; - std::string name() const override { return "decimal256"; } +/// \brief Concrete type class for decimal 128-bit data +class ARROW_EXPORT Decimal128Type : public BaseDecimalType<128> { + public: + static constexpr Type::type type_id = Type::DECIMAL128; + using BaseDecimalType<128>::BaseDecimalType; +}; - static constexpr int32_t kMinPrecision = 1; - static constexpr int32_t kMaxPrecision = 76; - static constexpr int32_t kByteWidth = 32; +/// \brief Concrete type class for decimal 256-bit data +class ARROW_EXPORT Decimal256Type : public BaseDecimalType<256> { + public: + static constexpr Type::type type_id = Type::DECIMAL256; + using BaseDecimalType<256>::BaseDecimalType; }; /// \brief Concrete type class for union data diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h index f1000d1fe7f..ca917cde7be 100644 --- a/cpp/src/arrow/type_fwd.h +++ b/cpp/src/arrow/type_fwd.h @@ -143,17 +143,39 @@ class StructArray; class StructBuilder; struct StructScalar; +class DecimalType; + +template +class DecimalAnyWidth; + +template +class BaseDecimalArray; + +template +class BaseDecimalBuilder; + +template +struct BaseDecimalScalar; + +using Decimal16 = DecimalAnyWidth<16>; +using Decimal32 = DecimalAnyWidth<32>; +using Decimal64 = DecimalAnyWidth<64>; class Decimal128; class Decimal256; -class DecimalType; -class Decimal128Type; -class Decimal256Type; -class Decimal128Array; -class Decimal256Array; -class Decimal128Builder; -class Decimal256Builder; -struct Decimal128Scalar; -struct Decimal256Scalar; + +#define DECIMAL_DECL(width) \ + class Decimal##width##Type; \ + using Decimal##width##Array = BaseDecimalArray; \ + using Decimal##width##Builder = BaseDecimalBuilder; \ + using Decimal##width##Scalar = BaseDecimalScalar; + +DECIMAL_DECL(16) +DECIMAL_DECL(32) +DECIMAL_DECL(64) +DECIMAL_DECL(128) +DECIMAL_DECL(256) + +#undef DECIMAL_DECL struct UnionMode { enum type { SPARSE, DENSE }; @@ -335,6 +357,12 @@ struct Type { /// DAY_TIME interval in SQL style INTERVAL_DAY_TIME, + DECIMAL16, + + DECIMAL32, + + DECIMAL64, + /// Precision- and scale-based decimal type with 128 bits. DECIMAL128, @@ -441,6 +469,18 @@ std::shared_ptr fixed_size_binary(int32_t byte_width); ARROW_EXPORT std::shared_ptr decimal(int32_t precision, int32_t scale); +/// \brief Create a Decimal16Type instance +ARROW_EXPORT +std::shared_ptr decimal16(int32_t precision, int32_t scale); + +/// \brief Create a Decimal32Type instance +ARROW_EXPORT +std::shared_ptr decimal32(int32_t precision, int32_t scale); + +/// \brief Create a Decimal64Type instance +ARROW_EXPORT +std::shared_ptr decimal64(int32_t precision, int32_t scale); + /// \brief Create a Decimal128Type instance ARROW_EXPORT std::shared_ptr decimal128(int32_t precision, int32_t scale); diff --git a/cpp/src/arrow/type_test.cc b/cpp/src/arrow/type_test.cc index d5ece2eea8e..3a150c5e91d 100644 --- a/cpp/src/arrow/type_test.cc +++ b/cpp/src/arrow/type_test.cc @@ -1851,6 +1851,48 @@ TEST(TypesTest, TestDecimal256Large) { EXPECT_EQ(t1.bit_width(), 256); } +TEST(TypesTest, TestDecimal16) { + Decimal16Type t1(5, 3); + + EXPECT_EQ(t1.id(), Type::DECIMAL16); + EXPECT_EQ(t1.precision(), 5); + EXPECT_EQ(t1.scale(), 3); + + EXPECT_EQ(t1.ToString(), std::string("decimal16(5, 3)")); + + // Test properties + EXPECT_EQ(t1.byte_width(), 2); + EXPECT_EQ(t1.bit_width(), 16); +} + +TEST(TypesTest, TestDecimal32) { + Decimal32Type t1(10, 5); + + EXPECT_EQ(t1.id(), Type::DECIMAL32); + EXPECT_EQ(t1.precision(), 10); + EXPECT_EQ(t1.scale(), 5); + + EXPECT_EQ(t1.ToString(), std::string("decimal32(10, 5)")); + + // Test properties + EXPECT_EQ(t1.byte_width(), 4); + EXPECT_EQ(t1.bit_width(), 32); +} + +TEST(TypesTest, TestDecimal64) { + Decimal64Type t1(19, 10); + + EXPECT_EQ(t1.id(), Type::DECIMAL64); + EXPECT_EQ(t1.precision(), 19); + EXPECT_EQ(t1.scale(), 10); + + EXPECT_EQ(t1.ToString(), std::string("decimal64(19, 10)")); + + // Test properties + EXPECT_EQ(t1.byte_width(), 8); + EXPECT_EQ(t1.bit_width(), 64); +} + TEST(TypesTest, TestDecimalEquals) { Decimal128Type t1(8, 4); Decimal128Type t2(8, 4); diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h index 2dcfc77c437..b1312219af0 100644 --- a/cpp/src/arrow/type_traits.h +++ b/cpp/src/arrow/type_traits.h @@ -66,6 +66,9 @@ TYPE_ID_TRAIT(TIMESTAMP, TimestampType) TYPE_ID_TRAIT(INTERVAL_DAY_TIME, DayTimeIntervalType) TYPE_ID_TRAIT(INTERVAL_MONTHS, MonthIntervalType) TYPE_ID_TRAIT(DURATION, DurationType) +TYPE_ID_TRAIT(DECIMAL16, Decimal16Type) +TYPE_ID_TRAIT(DECIMAL32, Decimal32Type) +TYPE_ID_TRAIT(DECIMAL64, Decimal64Type) TYPE_ID_TRAIT(DECIMAL128, Decimal128Type) TYPE_ID_TRAIT(DECIMAL256, Decimal256Type) TYPE_ID_TRAIT(STRUCT, StructType) @@ -281,21 +284,22 @@ struct TypeTraits { static inline std::shared_ptr type_singleton() { return float16(); } }; -template <> -struct TypeTraits { - using ArrayType = Decimal128Array; - using BuilderType = Decimal128Builder; - using ScalarType = Decimal128Scalar; - constexpr static bool is_parameter_free = false; -}; +#define DECIMAL_TYPE_TRAITS_DEF(width) \ + template <> \ + struct TypeTraits { \ + using ArrayType = Decimal##width##Array; \ + using BuilderType = Decimal##width##Builder; \ + using ScalarType = Decimal##width##Scalar; \ + constexpr static bool is_parameter_free = false; \ + }; -template <> -struct TypeTraits { - using ArrayType = Decimal256Array; - using BuilderType = Decimal256Builder; - using ScalarType = Decimal256Scalar; - constexpr static bool is_parameter_free = false; -}; +DECIMAL_TYPE_TRAITS_DEF(16) +DECIMAL_TYPE_TRAITS_DEF(32) +DECIMAL_TYPE_TRAITS_DEF(64) +DECIMAL_TYPE_TRAITS_DEF(128) +DECIMAL_TYPE_TRAITS_DEF(256) + +#undef DECIMAL_TYPE_TRAITS_DEF template <> struct TypeTraits { @@ -586,6 +590,24 @@ using is_decimal_type = std::is_base_of; template using enable_if_decimal = enable_if_t::value, R>; +template +using is_decimal16_type = std::is_base_of; + +template +using enable_if_decimal16 = enable_if_t::value, R>; + +template +using is_decimal32_type = std::is_base_of; + +template +using enable_if_decimal32 = enable_if_t::value, R>; + +template +using is_decimal64_type = std::is_base_of; + +template +using enable_if_decimal64 = enable_if_t::value, R>; + template using is_decimal128_type = std::is_base_of; @@ -915,6 +937,9 @@ static inline bool is_dictionary(Type::type type_id) { static inline bool is_fixed_size_binary(Type::type type_id) { switch (type_id) { + case Type::DECIMAL16: + case Type::DECIMAL32: + case Type::DECIMAL64: case Type::DECIMAL128: case Type::DECIMAL256: case Type::FIXED_SIZE_BINARY: diff --git a/cpp/src/arrow/util/basic_decimal.cc b/cpp/src/arrow/util/basic_decimal.cc index 78d5b15d1c0..e1ca749e0c2 100644 --- a/cpp/src/arrow/util/basic_decimal.cc +++ b/cpp/src/arrow/util/basic_decimal.cc @@ -28,6 +28,8 @@ #include #include "arrow/util/bit_util.h" +#include "arrow/util/decimal_meta.h" +#include "arrow/util/decimal_scale_multipliers.h" #include "arrow/util/int128_internal.h" #include "arrow/util/int_util_internal.h" #include "arrow/util/logging.h" @@ -37,88 +39,7 @@ namespace arrow { using internal::SafeLeftShift; using internal::SafeSignedAdd; - -static const BasicDecimal128 ScaleMultipliers[] = { - BasicDecimal128(1LL), - BasicDecimal128(10LL), - BasicDecimal128(100LL), - BasicDecimal128(1000LL), - BasicDecimal128(10000LL), - BasicDecimal128(100000LL), - BasicDecimal128(1000000LL), - BasicDecimal128(10000000LL), - BasicDecimal128(100000000LL), - BasicDecimal128(1000000000LL), - BasicDecimal128(10000000000LL), - BasicDecimal128(100000000000LL), - BasicDecimal128(1000000000000LL), - BasicDecimal128(10000000000000LL), - BasicDecimal128(100000000000000LL), - BasicDecimal128(1000000000000000LL), - BasicDecimal128(10000000000000000LL), - BasicDecimal128(100000000000000000LL), - BasicDecimal128(1000000000000000000LL), - BasicDecimal128(0LL, 10000000000000000000ULL), - BasicDecimal128(5LL, 7766279631452241920ULL), - BasicDecimal128(54LL, 3875820019684212736ULL), - BasicDecimal128(542LL, 1864712049423024128ULL), - BasicDecimal128(5421LL, 200376420520689664ULL), - BasicDecimal128(54210LL, 2003764205206896640ULL), - BasicDecimal128(542101LL, 1590897978359414784ULL), - BasicDecimal128(5421010LL, 15908979783594147840ULL), - BasicDecimal128(54210108LL, 11515845246265065472ULL), - BasicDecimal128(542101086LL, 4477988020393345024ULL), - BasicDecimal128(5421010862LL, 7886392056514347008ULL), - BasicDecimal128(54210108624LL, 5076944270305263616ULL), - BasicDecimal128(542101086242LL, 13875954555633532928ULL), - BasicDecimal128(5421010862427LL, 9632337040368467968ULL), - BasicDecimal128(54210108624275LL, 4089650035136921600ULL), - BasicDecimal128(542101086242752LL, 4003012203950112768ULL), - BasicDecimal128(5421010862427522LL, 3136633892082024448ULL), - BasicDecimal128(54210108624275221LL, 12919594847110692864ULL), - BasicDecimal128(542101086242752217LL, 68739955140067328ULL), - BasicDecimal128(5421010862427522170LL, 687399551400673280ULL)}; - -static const BasicDecimal128 ScaleMultipliersHalf[] = { - BasicDecimal128(0ULL), - BasicDecimal128(5ULL), - BasicDecimal128(50ULL), - BasicDecimal128(500ULL), - BasicDecimal128(5000ULL), - BasicDecimal128(50000ULL), - BasicDecimal128(500000ULL), - BasicDecimal128(5000000ULL), - BasicDecimal128(50000000ULL), - BasicDecimal128(500000000ULL), - BasicDecimal128(5000000000ULL), - BasicDecimal128(50000000000ULL), - BasicDecimal128(500000000000ULL), - BasicDecimal128(5000000000000ULL), - BasicDecimal128(50000000000000ULL), - BasicDecimal128(500000000000000ULL), - BasicDecimal128(5000000000000000ULL), - BasicDecimal128(50000000000000000ULL), - BasicDecimal128(500000000000000000ULL), - BasicDecimal128(5000000000000000000ULL), - BasicDecimal128(2LL, 13106511852580896768ULL), - BasicDecimal128(27LL, 1937910009842106368ULL), - BasicDecimal128(271LL, 932356024711512064ULL), - BasicDecimal128(2710LL, 9323560247115120640ULL), - BasicDecimal128(27105LL, 1001882102603448320ULL), - BasicDecimal128(271050LL, 10018821026034483200ULL), - BasicDecimal128(2710505LL, 7954489891797073920ULL), - BasicDecimal128(27105054LL, 5757922623132532736ULL), - BasicDecimal128(271050543LL, 2238994010196672512ULL), - BasicDecimal128(2710505431LL, 3943196028257173504ULL), - BasicDecimal128(27105054312LL, 2538472135152631808ULL), - BasicDecimal128(271050543121LL, 6937977277816766464ULL), - BasicDecimal128(2710505431213LL, 14039540557039009792ULL), - BasicDecimal128(27105054312137LL, 11268197054423236608ULL), - BasicDecimal128(271050543121376LL, 2001506101975056384ULL), - BasicDecimal128(2710505431213761LL, 1568316946041012224ULL), - BasicDecimal128(27105054312137610LL, 15683169460410122240ULL), - BasicDecimal128(271050543121376108LL, 9257742014424809472ULL), - BasicDecimal128(2710505431213761085LL, 343699775700336640ULL)}; +using internal::SafeSignedMultiply; static const BasicDecimal256 ScaleMultipliersDecimal256[] = { BasicDecimal256({1ULL, 0ULL, 0ULL, 0ULL}), @@ -232,7 +153,7 @@ static constexpr uint64_t kInt64Mask = 0xFFFFFFFFFFFFFFFF; static constexpr uint64_t kInt32Mask = 0xFFFFFFFF; #endif -// same as ScaleMultipliers[38] - 1 +// same as ScaleMultipliers128[38] - 1 static constexpr BasicDecimal128 kMaxValue = BasicDecimal128(5421010862427522170LL, 687399551400673280ULL - 1); @@ -281,8 +202,8 @@ BasicDecimal128 BasicDecimal128::Abs(const BasicDecimal128& in) { bool BasicDecimal128::FitsInPrecision(int32_t precision) const { DCHECK_GT(precision, 0); - DCHECK_LE(precision, 38); - return BasicDecimal128::Abs(*this) < ScaleMultipliers[precision]; + DCHECK_LE(precision, DecimalMeta<128>::max_precision); + return BasicDecimal128::Abs(*this) < ScaleMultipliers128[precision]; } BasicDecimal128& BasicDecimal128::operator+=(const BasicDecimal128& right) { @@ -950,44 +871,44 @@ DecimalStatus BasicDecimal128::Rescale(int32_t original_scale, int32_t new_scale void BasicDecimal128::GetWholeAndFraction(int scale, BasicDecimal128* whole, BasicDecimal128* fraction) const { DCHECK_GE(scale, 0); - DCHECK_LE(scale, 38); + DCHECK_LE(scale, DecimalMeta<128>::max_precision); - BasicDecimal128 multiplier(ScaleMultipliers[scale]); + BasicDecimal128 multiplier(ScaleMultipliers128[scale]); auto s = Divide(multiplier, whole, fraction); DCHECK_EQ(s, DecimalStatus::kSuccess); } const BasicDecimal128& BasicDecimal128::GetScaleMultiplier(int32_t scale) { DCHECK_GE(scale, 0); - DCHECK_LE(scale, 38); + DCHECK_LE(scale, DecimalMeta<128>::max_precision); - return ScaleMultipliers[scale]; + return ScaleMultipliers128[scale]; } const BasicDecimal128& BasicDecimal128::GetMaxValue() { return kMaxValue; } BasicDecimal128 BasicDecimal128::IncreaseScaleBy(int32_t increase_by) const { DCHECK_GE(increase_by, 0); - DCHECK_LE(increase_by, 38); + DCHECK_LE(increase_by, DecimalMeta<128>::max_precision); - return (*this) * ScaleMultipliers[increase_by]; + return (*this) * ScaleMultipliers128[increase_by]; } BasicDecimal128 BasicDecimal128::ReduceScaleBy(int32_t reduce_by, bool round) const { DCHECK_GE(reduce_by, 0); - DCHECK_LE(reduce_by, 38); + DCHECK_LE(reduce_by, DecimalMeta<128>::max_precision); if (reduce_by == 0) { return *this; } - BasicDecimal128 divisor(ScaleMultipliers[reduce_by]); + BasicDecimal128 divisor(ScaleMultipliers128[reduce_by]); BasicDecimal128 result; BasicDecimal128 remainder; auto s = Divide(divisor, &result, &remainder); DCHECK_EQ(s, DecimalStatus::kSuccess); if (round) { - auto divisor_half = ScaleMultipliersHalf[reduce_by]; + auto divisor_half = ScaleMultipliersHalf128[reduce_by]; if (remainder.Abs() >= divisor_half) { if (result > 0) { result += 1; @@ -1195,4 +1116,151 @@ BasicDecimal256 operator/(const BasicDecimal256& left, const BasicDecimal256& ri return result; } +/// BasicDecimalAnyWidth + +template +BasicDecimalAnyWidth::BasicDecimalAnyWidth(const uint8_t* bytes) { + DCHECK_NE(bytes, nullptr); + value = *(reinterpret_cast(bytes)); +} + +template +BasicDecimalAnyWidth& BasicDecimalAnyWidth::operator+=( + const BasicDecimalAnyWidth& right) { + value = SafeSignedAdd(value, right.value); + return *this; +} + +template +BasicDecimalAnyWidth& BasicDecimalAnyWidth::operator-=( + const BasicDecimalAnyWidth& right) { + value -= right.value; + return *this; +} + +template +BasicDecimalAnyWidth& BasicDecimalAnyWidth::operator*=( + const BasicDecimalAnyWidth& right) { + value = SafeSignedMultiply(value, right.value); + return *this; +} + +template +BasicDecimalAnyWidth& BasicDecimalAnyWidth::operator/=( + const BasicDecimalAnyWidth& right) { + BasicDecimalAnyWidth remainder; + auto s = Divide(right, this, &remainder); + DCHECK_EQ(s, DecimalStatus::kSuccess); + return *this; +} + +template +BasicDecimalAnyWidth& BasicDecimalAnyWidth::operator%=( + const BasicDecimalAnyWidth& right) { + BasicDecimalAnyWidth result; + auto s = Divide(right, &result, this); + DCHECK_EQ(s, DecimalStatus::kSuccess); + return *this; +} + +template +BasicDecimalAnyWidth& BasicDecimalAnyWidth::Abs() { + return *this < 0 ? Negate() : *this; +} + +template +BasicDecimalAnyWidth BasicDecimalAnyWidth::Abs( + const BasicDecimalAnyWidth& in) { + BasicDecimalAnyWidth result(in); + return result.Abs(); +} + +template +DecimalStatus BasicDecimalAnyWidth::Divide(const BasicDecimalAnyWidth& divisor, + BasicDecimalAnyWidth* result, + BasicDecimalAnyWidth* remainder) const { + if (divisor.value == 0) { + return DecimalStatus::kDivideByZero; + } + + bool dividen_was_negative = Sign() == -1; + bool divisor_was_negative = divisor.Sign() == -1; + + BasicDecimalAnyWidth dividen_abs = BasicDecimalAnyWidth::Abs(*this); + BasicDecimalAnyWidth divisor_abs = BasicDecimalAnyWidth::Abs(divisor); + + *result = dividen_abs.value / divisor_abs.value; + *remainder = dividen_abs.value % divisor_abs.value; + + FixDivisionSigns(result, remainder, dividen_was_negative, divisor_was_negative); + return DecimalStatus::kSuccess; +} + +template +BasicDecimalAnyWidth BasicDecimalAnyWidth::GetScaleMultiplier( + int32_t scale) { + DCHECK_GE(scale, 0); + DCHECK_LE(scale, DecimalMeta::max_precision); + + return BasicDecimalAnyWidth(ScaleMultipliersAnyWidth::value[scale]); +} + +template +std::array> 3)> BasicDecimalAnyWidth::ToBytes() const { + std::array> 3)> out{{0}}; + ToBytes(out.data()); + return out; +} + +template +void BasicDecimalAnyWidth::ToBytes(uint8_t* out) const { + DCHECK_NE(out, nullptr); + reinterpret_cast(out)[0] = value; +} + +template +BasicDecimalAnyWidth& BasicDecimalAnyWidth::Negate() { + value = -value; + return *this; +} + +template +DecimalStatus BasicDecimalAnyWidth::Rescale( + int32_t original_scale, int32_t new_scale, BasicDecimalAnyWidth* out) const { + return DecimalRescale(*this, original_scale, new_scale, out); +} + +template +bool BasicDecimalAnyWidth::FitsInPrecision(int32_t precision) const { + DCHECK_GT(precision, 0); + DCHECK_LE(precision, DecimalMeta::max_precision); + return BasicDecimalAnyWidth::Abs(*this) < + ScaleMultipliersAnyWidth::value[precision]; +} + +template +void BasicDecimalAnyWidth::GetWholeAndFraction( + int scale, BasicDecimalAnyWidth* whole, + BasicDecimalAnyWidth* fraction) const { + DCHECK_GE(scale, 0); + DCHECK_LE(scale, DecimalMeta::max_precision); + + BasicDecimalAnyWidth multiplier(ScaleMultipliersAnyWidth::value[scale]); + auto s = Divide(multiplier, whole, fraction); + DCHECK_EQ(s, DecimalStatus::kSuccess); +} + +template +BasicDecimalAnyWidth BasicDecimalAnyWidth::IncreaseScaleBy( + int32_t increase_by) const { + DCHECK_GE(increase_by, 0); + DCHECK_LE(increase_by, DecimalMeta::max_precision); + + return (*this) * ScaleMultipliersAnyWidth::value[increase_by]; +} + +template class ARROW_EXPORT BasicDecimalAnyWidth<64>; +template class ARROW_EXPORT BasicDecimalAnyWidth<32>; +template class ARROW_EXPORT BasicDecimalAnyWidth<16>; + } // namespace arrow diff --git a/cpp/src/arrow/util/basic_decimal.h b/cpp/src/arrow/util/basic_decimal.h index b62d8942077..2e458c46b69 100644 --- a/cpp/src/arrow/util/basic_decimal.h +++ b/cpp/src/arrow/util/basic_decimal.h @@ -23,6 +23,7 @@ #include #include +#include "arrow/util/decimal_meta.h" #include "arrow/util/macros.h" #include "arrow/util/type_traits.h" #include "arrow/util/visibility.h" @@ -36,6 +37,9 @@ enum class DecimalStatus { kRescaleDataLoss, }; +template +class BasicDecimalAnyWidth; + /// Represents a signed 128-bit integer in two's complement. /// /// This class is also compiled into LLVM IR - so, it should not have cpp references like @@ -59,6 +63,11 @@ class ARROW_EXPORT BasicDecimal128 { : BasicDecimal128(value >= T{0} ? 0 : -1, static_cast(value)) { // NOLINT } + /// \brief Upcast BasicDecimal with less widths + template + constexpr BasicDecimal128(const BasicDecimalAnyWidth& other) noexcept + : BasicDecimal128(other.Value()) {} + /// \brief Create a BasicDecimal128 from an array of bytes. Bytes are assumed to be in /// native-endian byte order. explicit BasicDecimal128(const uint8_t* bytes); @@ -211,6 +220,10 @@ class ARROW_EXPORT BasicDecimal256 { : little_endian_array_({value.low_bits(), static_cast(value.high_bits()), extend(value.high_bits()), extend(value.high_bits())}) {} + template + constexpr BasicDecimal256(const BasicDecimalAnyWidth& other) noexcept + : BasicDecimal256(other.Value()) {} + /// \brief Create a BasicDecimal256 from an array of bytes. Bytes are assumed to be in /// native-endian byte order. explicit BasicDecimal256(const uint8_t* bytes); @@ -322,4 +335,164 @@ ARROW_EXPORT BasicDecimal256 operator*(const BasicDecimal256& left, const BasicDecimal256& right); ARROW_EXPORT BasicDecimal256 operator/(const BasicDecimal256& left, const BasicDecimal256& right); + +template +class ARROW_EXPORT BasicDecimalAnyWidth { + public: + static constexpr int bit_width = width; + using ValueType = typename IntTypes::signed_type; + /// \brief Empty constructor creates a BasicDecimal with a value of 0. + constexpr BasicDecimalAnyWidth() noexcept : value(0) {} + + /// \brief Convert any integer value into a BasicDecimal. + template ::value && + ((sizeof(T) < sizeof(ValueType)) || + ((sizeof(T) == sizeof(ValueType)) && + std::is_signed::value) || + std::is_same::value), + T>::type> + constexpr BasicDecimalAnyWidth(T value) noexcept + : value(static_cast(value)) {} + + /// \brief Create a BasicDecimal from an array of bytes. Bytes are assumed to be in + /// native-endian byte order. + explicit BasicDecimalAnyWidth(const uint8_t* bytes); + + /// \brief Upcast BasicDecimal with less widths + template ::type> + constexpr BasicDecimalAnyWidth(const BasicDecimalAnyWidth<_width>& other) noexcept + : value(static_cast(other.Value())) {} + + /// \brief Negate the current value (in-place) + BasicDecimalAnyWidth& Negate(); + + /// \brief Absolute value (in-place) + BasicDecimalAnyWidth& Abs(); + + /// \brief Absolute value + static BasicDecimalAnyWidth Abs(const BasicDecimalAnyWidth& left); + + /// Divide this number by right and return the result. + /// + /// This operation is not destructive. + /// The answer rounds to zero. Signs work like: + /// 21 / 5 -> 4, 1 + /// -21 / 5 -> -4, -1 + /// 21 / -5 -> -4, 1 + /// -21 / -5 -> 4, -1 + /// \param[in] divisor the number to divide by + /// \param[out] result the quotient + /// \param[out] remainder the remainder after the division + DecimalStatus Divide(const BasicDecimalAnyWidth& divisor, BasicDecimalAnyWidth* result, + BasicDecimalAnyWidth* remainder) const; + + /// \brief Scale multiplier for given scale value. + static BasicDecimalAnyWidth GetScaleMultiplier(int32_t scale); + + /// \brief Return the raw bytes of the value in native-endian byte order. + std::array> 3)> ToBytes() const; + void ToBytes(uint8_t* out) const; + + /// \brief Convert BasicDecimal from one scale to another + DecimalStatus Rescale(int32_t original_scale, int32_t new_scale, + BasicDecimalAnyWidth* out) const; + + inline int64_t Sign() const { return value >= 0 ? 1 : -1; } + + /// \brief Get the high bits of the two's complement representation of the number. + inline constexpr ValueType Value() const { return value; } + + /// \brief Whether this number fits in the given precision + /// + /// Return true if the number of significant digits is less or equal to `precision`. + bool FitsInPrecision(int32_t precision) const; + + /// \brief separate the integer and fractional parts for the given scale. + void GetWholeAndFraction(int32_t scale, BasicDecimalAnyWidth* whole, + BasicDecimalAnyWidth* fraction) const; + + /// \brief Scale up. + BasicDecimalAnyWidth IncreaseScaleBy(int32_t increase_by) const; + + BasicDecimalAnyWidth& operator+=(const BasicDecimalAnyWidth&); + BasicDecimalAnyWidth& operator-=(const BasicDecimalAnyWidth&); + BasicDecimalAnyWidth& operator*=(const BasicDecimalAnyWidth&); + BasicDecimalAnyWidth& operator/=(const BasicDecimalAnyWidth&); + BasicDecimalAnyWidth& operator%=(const BasicDecimalAnyWidth&); + + friend bool operator==(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + return left.value == right.value; + } + + friend bool operator!=(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + return !operator==(left, right); + } + + friend bool operator<(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + return left.value < right.value; + } + + friend bool operator<=(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + return !operator<(right, left); + } + + friend bool operator>(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + return operator<(right, left); + } + + friend bool operator>=(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + return !operator<(left, right); + } + + friend BasicDecimalAnyWidth operator+(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + BasicDecimalAnyWidth result(left); + result += right; + return result; + } + + friend BasicDecimalAnyWidth operator-(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + BasicDecimalAnyWidth result(left); + result -= right; + return result; + } + + friend BasicDecimalAnyWidth operator*(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + BasicDecimalAnyWidth result(left); + result *= right; + return result; + } + + friend BasicDecimalAnyWidth operator/(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + BasicDecimalAnyWidth result = left; + result /= right; + return result; + } + + friend BasicDecimalAnyWidth operator%(const BasicDecimalAnyWidth& left, + const BasicDecimalAnyWidth& right) { + BasicDecimalAnyWidth result = left; + result %= right; + return result; + } + + private: + ValueType value; +}; + +using BasicDecimal64 = BasicDecimalAnyWidth<64>; +using BasicDecimal32 = BasicDecimalAnyWidth<32>; +using BasicDecimal16 = BasicDecimalAnyWidth<16>; + } // namespace arrow diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc index dcb2023616a..35566f27428 100644 --- a/cpp/src/arrow/util/decimal.cc +++ b/cpp/src/arrow/util/decimal.cc @@ -463,15 +463,13 @@ inline Status ToArrowStatus(DecimalStatus dstatus, int num_bits) { return Status::OK(); } -} // namespace - -Status Decimal128::FromString(const util::string_view& s, Decimal128* out, - int32_t* precision, int32_t* scale) { +Status FromStringToArray(const util::string_view& s, DecimalComponents& dec, + uint64_t* out, int32_t array_size, int32_t* precision, + int32_t* scale) { if (s.empty()) { return Status::Invalid("Empty string cannot be converted to decimal"); } - DecimalComponents dec; if (!ParseDecimalComponents(s.data(), s.size(), &dec)) { return Status::Invalid("The string '", s, "' is not a valid decimal number"); } @@ -482,26 +480,48 @@ Status Decimal128::FromString(const util::string_view& s, Decimal128* out, if (first_non_zero != std::string::npos) { significant_digits += dec.whole_digits.size() - first_non_zero; } - int32_t parsed_precision = static_cast(significant_digits); - int32_t parsed_scale = 0; - if (dec.has_exponent) { - auto adjusted_exponent = dec.exponent; - auto len = static_cast(significant_digits); - parsed_scale = -adjusted_exponent + len - 1; - } else { - parsed_scale = static_cast(dec.fractional_digits.size()); + if (precision != nullptr) { + *precision = static_cast(significant_digits); + } + + if (scale != nullptr) { + if (dec.has_exponent) { + auto adjusted_exponent = dec.exponent; + auto len = static_cast(significant_digits); + *scale = -adjusted_exponent + len - 1; + } else { + *scale = static_cast(dec.fractional_digits.size()); + } + } + + if (out != nullptr) { + ShiftAndAdd(dec.whole_digits, out, array_size); + ShiftAndAdd(dec.fractional_digits, out, array_size); + } + + return Status::OK(); +} + +} // namespace + +Status Decimal128::FromString(const util::string_view& s, Decimal128* out, + int32_t* precision, int32_t* scale) { + std::array little_endian_array = {0, 0}; + DecimalComponents dec; + + auto status = + FromStringToArray(s, dec, little_endian_array.data(), 2, precision, scale); + if (status != Status::OK()) { + return status; } if (out != nullptr) { - std::array little_endian_array = {0, 0}; - ShiftAndAdd(dec.whole_digits, little_endian_array.data(), little_endian_array.size()); - ShiftAndAdd(dec.fractional_digits, little_endian_array.data(), - little_endian_array.size()); *out = Decimal128(static_cast(little_endian_array[1]), little_endian_array[0]); - if (parsed_scale < 0) { - *out *= GetScaleMultiplier(-parsed_scale); + + if (scale != nullptr && *scale < 0) { + *out *= GetScaleMultiplier(-*scale); } if (dec.sign == '-') { @@ -509,19 +529,14 @@ Status Decimal128::FromString(const util::string_view& s, Decimal128* out, } } - if (parsed_scale < 0) { - parsed_precision -= parsed_scale; - parsed_scale = 0; - } - - if (precision != nullptr) { - *precision = parsed_precision; - } - if (scale != nullptr) { - *scale = parsed_scale; + if (scale != nullptr && *scale < 0) { + if (precision != nullptr) { + *precision -= *scale; + } + *scale = 0; } - return Status::OK(); + return status; } Status Decimal128::FromString(const std::string& s, Decimal128* out, int32_t* precision, @@ -649,49 +664,23 @@ std::string Decimal256::ToString(int32_t scale) const { Status Decimal256::FromString(const util::string_view& s, Decimal256* out, int32_t* precision, int32_t* scale) { - if (s.empty()) { - return Status::Invalid("Empty string cannot be converted to decimal"); - } - + std::array little_endian_array = {0, 0, 0, 0}; DecimalComponents dec; - if (!ParseDecimalComponents(s.data(), s.size(), &dec)) { - return Status::Invalid("The string '", s, "' is not a valid decimal number"); - } - - // Count number of significant digits (without leading zeros) - size_t first_non_zero = dec.whole_digits.find_first_not_of('0'); - size_t significant_digits = dec.fractional_digits.size(); - if (first_non_zero != std::string::npos) { - significant_digits += dec.whole_digits.size() - first_non_zero; - } - if (precision != nullptr) { - *precision = static_cast(significant_digits); - } - - if (scale != nullptr) { - if (dec.has_exponent) { - auto adjusted_exponent = dec.exponent; - auto len = static_cast(significant_digits); - *scale = -adjusted_exponent + len - 1; - } else { - *scale = static_cast(dec.fractional_digits.size()); - } + auto status = + FromStringToArray(s, dec, little_endian_array.data(), 4, precision, scale); + if (status != Status::OK()) { + return status; } if (out != nullptr) { - std::array little_endian_array = {0, 0, 0, 0}; - ShiftAndAdd(dec.whole_digits, little_endian_array.data(), little_endian_array.size()); - ShiftAndAdd(dec.fractional_digits, little_endian_array.data(), - little_endian_array.size()); *out = Decimal256(little_endian_array); - if (dec.sign == '-') { out->Negate(); } } - return Status::OK(); + return status; } Status Decimal256::FromString(const std::string& s, Decimal256* out, int32_t* precision, @@ -768,4 +757,101 @@ std::ostream& operator<<(std::ostream& os, const Decimal256& decimal) { os << decimal.ToIntegerString(); return os; } + +template +DecimalAnyWidth::DecimalAnyWidth(const std::string& str) : DecimalAnyWidth() { + *this = DecimalAnyWidth::FromString(str).ValueOrDie(); +} + +template +std::string DecimalAnyWidth::ToIntegerString() const { + std::stringstream ss; + ss << this->Value(); + return ss.str(); +} + +template +std::string DecimalAnyWidth::ToString(int32_t scale) const { + std::string str(ToIntegerString()); + AdjustIntegerStringWithScale(scale, &str); + return str; +} + +template +Status DecimalAnyWidth::FromString(const util::string_view& s, + DecimalAnyWidth* out, int32_t* precision, + int32_t* scale) { + std::array little_endian_array = {0}; + DecimalComponents dec; + + auto status = + FromStringToArray(s, dec, little_endian_array.data(), 1, precision, scale); + if (status != Status::OK()) { + return status; + } + + if (out != nullptr) { + *out = DecimalAnyWidth(static_cast(little_endian_array[0])); + + if (scale != nullptr && *scale < 0) { + *out *= BasicDecimalAnyWidth::GetScaleMultiplier(*scale); + } + + if (dec.sign == '-') { + out->Negate(); + } + } + + if (scale != nullptr && *scale < 0) { + if (precision != nullptr) { + *precision -= *scale; + } + *scale = 0; + } + + return status; +} + +template +Status DecimalAnyWidth::FromString(const std::string& s, + DecimalAnyWidth* out, int32_t* precision, + int32_t* scale) { + return FromString(util::string_view(s), out, precision, scale); +} + +template +Status DecimalAnyWidth::FromString(const char* s, DecimalAnyWidth* out, + int32_t* precision, int32_t* scale) { + return FromString(util::string_view(s), out, precision, scale); +} + +template +Result::_DecimalType> DecimalAnyWidth::FromString( + const util::string_view& s) { + _DecimalType out; + RETURN_NOT_OK(FromString(s, &out, nullptr, nullptr)); + return std::move(out); +} + +template +Result::_DecimalType> DecimalAnyWidth::FromString( + const std::string& s) { + return FromString(util::string_view(s)); +} + +template +Result::_DecimalType> DecimalAnyWidth::FromString( + const char* s) { + return FromString(util::string_view(s)); +} + +template +Status DecimalAnyWidth::ToArrowStatus(DecimalStatus dstatus) const { + return arrow::ToArrowStatus(dstatus, width); +} + +template class ARROW_EXPORT DecimalAnyWidth<16>; +template class ARROW_EXPORT DecimalAnyWidth<32>; +template class ARROW_EXPORT DecimalAnyWidth<64>; + } // namespace arrow diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h index 3d41ae460e4..e8cec7f1f68 100644 --- a/cpp/src/arrow/util/decimal.h +++ b/cpp/src/arrow/util/decimal.h @@ -26,6 +26,7 @@ #include "arrow/result.h" #include "arrow/status.h" #include "arrow/util/basic_decimal.h" +#include "arrow/util/decimal_type_traits.h" #include "arrow/util/string_view.h" namespace arrow { @@ -257,4 +258,63 @@ class ARROW_EXPORT Decimal256 : public BasicDecimal256 { Status ToArrowStatus(DecimalStatus dstatus) const; }; +template +class ARROW_EXPORT DecimalAnyWidth : public BasicDecimalAnyWidth { + public: + using _DecimalType = typename DecimalTypeTraits::ValueType; + using ValueType = typename BasicDecimalAnyWidth::ValueType; + + /// \cond FALSE + // (need to avoid a duplicate definition in Sphinx) + using BasicDecimalAnyWidth::BasicDecimalAnyWidth; + /// \endcond + + /// \brief constructor creates a Decimal256 from a BasicDecimal128. + constexpr DecimalAnyWidth(const BasicDecimalAnyWidth& value) noexcept + : BasicDecimalAnyWidth(value) {} + + /// \brief Parse the number from a base 10 string representation. + explicit DecimalAnyWidth(const std::string& value); + + /// \brief Empty constructor creates a Decimal256 with a value of 0. + // This is required on some older compilers. + constexpr DecimalAnyWidth() noexcept : BasicDecimalAnyWidth() {} + + /// \brief Convert the Decimal256 value to a base 10 decimal string with the given + /// scale. + std::string ToString(int32_t scale) const; + + /// \brief Convert the value to an integer string + std::string ToIntegerString() const; + + /// \brief Convert a decimal string to a Decimal256 value, optionally including + /// precision and scale if they're passed in and not null. + static Status FromString(const util::string_view& s, DecimalAnyWidth* out, + int32_t* precision, int32_t* scale = NULLPTR); + static Status FromString(const std::string& s, DecimalAnyWidth* out, int32_t* precision, + int32_t* scale = NULLPTR); + static Status FromString(const char* s, DecimalAnyWidth* out, int32_t* precision, + int32_t* scale = NULLPTR); + static Result<_DecimalType> FromString(const util::string_view& s); + static Result<_DecimalType> FromString(const std::string& s); + static Result<_DecimalType> FromString(const char* s); + + /// \brief Convert Decimal256 from one scale to another + Result<_DecimalType> Rescale(int32_t original_scale, int32_t new_scale) const { + _DecimalType out; + auto dstatus = BasicDecimalAnyWidth::Rescale(original_scale, new_scale, &out); + ARROW_RETURN_NOT_OK(ToArrowStatus(dstatus)); + return std::move(out); + } + + friend std::ostream& operator<<(std::ostream& os, const DecimalAnyWidth& decimal) { + os << decimal.ToIntegerString(); + return os; + } + + private: + /// Converts internal error code to Status + Status ToArrowStatus(DecimalStatus dstatus) const; +}; + } // namespace arrow diff --git a/cpp/src/arrow/util/decimal_meta.h b/cpp/src/arrow/util/decimal_meta.h new file mode 100644 index 00000000000..727890926fd --- /dev/null +++ b/cpp/src/arrow/util/decimal_meta.h @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace arrow { + +template +struct IntTypes {}; + +#define IntTypes_DECL(bit_width) \ + template <> \ + struct IntTypes { \ + using signed_type = int##bit_width##_t; \ + using unsigned_type = uint##bit_width##_t; \ + }; + +IntTypes_DECL(64); +IntTypes_DECL(32); +IntTypes_DECL(16); + +template +struct DecimalMeta; + +template <> +struct DecimalMeta<16> { + static constexpr const char* name = "decimal16"; + static constexpr int32_t max_precision = 5; +}; + +template <> +struct DecimalMeta<32> { + static constexpr const char* name = "decimal32"; + static constexpr int32_t max_precision = 10; +}; + +template <> +struct DecimalMeta<64> { + static constexpr const char* name = "decimal64"; + static constexpr int32_t max_precision = 19; +}; + +template <> +struct DecimalMeta<128> { + static constexpr const char* name = "decimal"; + static constexpr int32_t max_precision = 38; +}; + +template <> +struct DecimalMeta<256> { + static constexpr const char* name = "decimal256"; + static constexpr int32_t max_precision = 76; +}; + +} // namespace arrow diff --git a/cpp/src/arrow/util/decimal_scale_multipliers.h b/cpp/src/arrow/util/decimal_scale_multipliers.h new file mode 100644 index 00000000000..0efb9d8784f --- /dev/null +++ b/cpp/src/arrow/util/decimal_scale_multipliers.h @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/util/basic_decimal.h" + +namespace arrow { + +template +struct ScaleMultipliersAnyWidth {}; + +#define DECL_ANY_SCALE_MULTIPLIERS(width) \ + template <> \ + struct ScaleMultipliersAnyWidth { \ + static const int##width##_t value[]; \ + }; \ + const int##width##_t ScaleMultipliersAnyWidth::value[] = { \ + int##width##_t(1LL), \ + int##width##_t(10LL), \ + int##width##_t(100LL), \ + int##width##_t(1000LL), \ + int##width##_t(10000LL), \ + int##width##_t(100000LL), \ + int##width##_t(1000000LL), \ + int##width##_t(10000000LL), \ + int##width##_t(100000000LL), \ + int##width##_t(1000000000LL), \ + int##width##_t(10000000000LL), \ + int##width##_t(100000000000LL), \ + int##width##_t(1000000000000LL), \ + int##width##_t(10000000000000LL), \ + int##width##_t(100000000000000LL), \ + int##width##_t(1000000000000000LL), \ + int##width##_t(10000000000000000LL), \ + int##width##_t(100000000000000000LL), \ + int##width##_t(1000000000000000000LL)}; + +DECL_ANY_SCALE_MULTIPLIERS(16) +DECL_ANY_SCALE_MULTIPLIERS(32) +DECL_ANY_SCALE_MULTIPLIERS(64) + +#undef DECL_ANY_SCALE_MULTIPLIERS + +static const BasicDecimal128 ScaleMultipliers128[] = { + BasicDecimal128(1LL), + BasicDecimal128(10LL), + BasicDecimal128(100LL), + BasicDecimal128(1000LL), + BasicDecimal128(10000LL), + BasicDecimal128(100000LL), + BasicDecimal128(1000000LL), + BasicDecimal128(10000000LL), + BasicDecimal128(100000000LL), + BasicDecimal128(1000000000LL), + BasicDecimal128(10000000000LL), + BasicDecimal128(100000000000LL), + BasicDecimal128(1000000000000LL), + BasicDecimal128(10000000000000LL), + BasicDecimal128(100000000000000LL), + BasicDecimal128(1000000000000000LL), + BasicDecimal128(10000000000000000LL), + BasicDecimal128(100000000000000000LL), + BasicDecimal128(1000000000000000000LL), + BasicDecimal128(0LL, 10000000000000000000ULL), + BasicDecimal128(5LL, 7766279631452241920ULL), + BasicDecimal128(54LL, 3875820019684212736ULL), + BasicDecimal128(542LL, 1864712049423024128ULL), + BasicDecimal128(5421LL, 200376420520689664ULL), + BasicDecimal128(54210LL, 2003764205206896640ULL), + BasicDecimal128(542101LL, 1590897978359414784ULL), + BasicDecimal128(5421010LL, 15908979783594147840ULL), + BasicDecimal128(54210108LL, 11515845246265065472ULL), + BasicDecimal128(542101086LL, 4477988020393345024ULL), + BasicDecimal128(5421010862LL, 7886392056514347008ULL), + BasicDecimal128(54210108624LL, 5076944270305263616ULL), + BasicDecimal128(542101086242LL, 13875954555633532928ULL), + BasicDecimal128(5421010862427LL, 9632337040368467968ULL), + BasicDecimal128(54210108624275LL, 4089650035136921600ULL), + BasicDecimal128(542101086242752LL, 4003012203950112768ULL), + BasicDecimal128(5421010862427522LL, 3136633892082024448ULL), + BasicDecimal128(54210108624275221LL, 12919594847110692864ULL), + BasicDecimal128(542101086242752217LL, 68739955140067328ULL), + BasicDecimal128(5421010862427522170LL, 687399551400673280ULL)}; + +static const BasicDecimal128 ScaleMultipliersHalf128[] = { + BasicDecimal128(0ULL), + BasicDecimal128(5ULL), + BasicDecimal128(50ULL), + BasicDecimal128(500ULL), + BasicDecimal128(5000ULL), + BasicDecimal128(50000ULL), + BasicDecimal128(500000ULL), + BasicDecimal128(5000000ULL), + BasicDecimal128(50000000ULL), + BasicDecimal128(500000000ULL), + BasicDecimal128(5000000000ULL), + BasicDecimal128(50000000000ULL), + BasicDecimal128(500000000000ULL), + BasicDecimal128(5000000000000ULL), + BasicDecimal128(50000000000000ULL), + BasicDecimal128(500000000000000ULL), + BasicDecimal128(5000000000000000ULL), + BasicDecimal128(50000000000000000ULL), + BasicDecimal128(500000000000000000ULL), + BasicDecimal128(5000000000000000000ULL), + BasicDecimal128(2LL, 13106511852580896768ULL), + BasicDecimal128(27LL, 1937910009842106368ULL), + BasicDecimal128(271LL, 932356024711512064ULL), + BasicDecimal128(2710LL, 9323560247115120640ULL), + BasicDecimal128(27105LL, 1001882102603448320ULL), + BasicDecimal128(271050LL, 10018821026034483200ULL), + BasicDecimal128(2710505LL, 7954489891797073920ULL), + BasicDecimal128(27105054LL, 5757922623132532736ULL), + BasicDecimal128(271050543LL, 2238994010196672512ULL), + BasicDecimal128(2710505431LL, 3943196028257173504ULL), + BasicDecimal128(27105054312LL, 2538472135152631808ULL), + BasicDecimal128(271050543121LL, 6937977277816766464ULL), + BasicDecimal128(2710505431213LL, 14039540557039009792ULL), + BasicDecimal128(27105054312137LL, 11268197054423236608ULL), + BasicDecimal128(271050543121376LL, 2001506101975056384ULL), + BasicDecimal128(2710505431213761LL, 1568316946041012224ULL), + BasicDecimal128(27105054312137610LL, 15683169460410122240ULL), + BasicDecimal128(271050543121376108LL, 9257742014424809472ULL), + BasicDecimal128(2710505431213761085LL, 343699775700336640ULL)}; + +} // namespace arrow diff --git a/cpp/src/arrow/util/decimal_test.cc b/cpp/src/arrow/util/decimal_test.cc index 40ae49da2ce..95bb3ac0c8a 100644 --- a/cpp/src/arrow/util/decimal_test.cc +++ b/cpp/src/arrow/util/decimal_test.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -899,7 +900,6 @@ std::vector GetRandomNumbers(int32_t size) { auto rand = random::RandomArrayGenerator(0x5487655); auto x_array = rand.Numeric(size, static_cast(0), std::numeric_limits::max(), 0); - auto x_ptr = x_array->data()->template GetValues(1); std::vector ret; for (int i = 0; i < size; ++i) { @@ -1567,4 +1567,185 @@ TEST_P(Decimal256ToStringTest, ToString) { INSTANTIATE_TEST_SUITE_P(Decimal256ToStringTest, Decimal256ToStringTest, ::testing::ValuesIn(kToStringTestData)); +// DecimalAnyWidth + +template +class DecimalAnyWidthTest : public ::testing::Test {}; + +template +class Decimal16Test : public ::testing::Test {}; + +template +class Decimal32Test : public ::testing::Test {}; + +template +class Decimal64Test : public ::testing::Test {}; + +using DecimalTypes = ::testing::Types; + +struct DecimalFromStringParams { + std::string value; + int expected_value; + int32_t expected_scale; + int32_t expected_precision; +}; + +static const std::vector DecimalFromStringParamsList = { + {"1234", 1234, 0, 4}, + {"12.34", 1234, 2, 4}, + {"+12.34", 1234, 2, 4}, + {"-12.34", -1234, 2, 4}, + {".0000", 0, 4, 4}}; + +TYPED_TEST_SUITE(DecimalAnyWidthTest, DecimalTypes); + +TYPED_TEST(DecimalAnyWidthTest, FromString) { + for (auto& param : DecimalFromStringParamsList) { + TypeParam d; + int precision, scale; + + ASSERT_OK(TypeParam::FromString(param.value, &d, &precision, &scale)); + + ASSERT_EQ(param.expected_value, d); + ASSERT_EQ(param.expected_precision, precision); + ASSERT_EQ(param.expected_scale, scale); + } +} + +TYPED_TEST(DecimalAnyWidthTest, FromBool) { + ASSERT_EQ(TypeParam(0), TypeParam(false)); + ASSERT_EQ(TypeParam(1), TypeParam(true)); +} + +using Decimal16Types = ::testing::Types; // NOLINT + +using Decimal32Types = + ::testing::Types; // NOLINT + +using Decimal64Types = + ::testing::Types; // NOLINT + +TYPED_TEST_SUITE(Decimal16Test, Decimal16Types); + +TYPED_TEST(Decimal16Test, Decimal16Types) { + TypeParam value = 42; + TypeParam max_value = std::numeric_limits::max(); + TypeParam min_value = std::numeric_limits::min(); + + Decimal16 d(value); + ASSERT_EQ(value, d); + + // Constructing from int will cause overflow + if (std::is_same::value) { + Decimal16 max_value_d(max_value); + ASSERT_EQ(static_cast(max_value), max_value_d); + + Decimal16 min_value_d(min_value); + ASSERT_EQ(static_cast(min_value), min_value_d); + } else { + Decimal16 max_value_d(max_value); + ASSERT_EQ(max_value, max_value_d); + + Decimal16 min_value_d(min_value); + ASSERT_EQ(min_value, min_value_d); + } +} + +TYPED_TEST_SUITE(Decimal32Test, Decimal32Types); + +TYPED_TEST(Decimal32Test, Decimal32Types) { + TypeParam value = 42; + TypeParam max_value = std::numeric_limits::max(); + TypeParam min_value = std::numeric_limits::min(); + + Decimal32 d(value); + ASSERT_EQ(value, d); + + Decimal32 max_value_d(max_value); + ASSERT_EQ(max_value, max_value_d); + + Decimal32 min_value_d(min_value); + ASSERT_EQ(min_value, min_value_d); +} + +TYPED_TEST_SUITE(Decimal64Test, Decimal64Types); + +TYPED_TEST(Decimal64Test, Decimal64Types) { + TypeParam value = 42; + TypeParam max_value = std::numeric_limits::max(); + TypeParam min_value = std::numeric_limits::min(); + + Decimal64 d(value); + ASSERT_EQ(value, d); + + Decimal64 max_value_d(max_value); + ASSERT_EQ(max_value, max_value_d); + + Decimal64 min_value_d(min_value); + ASSERT_EQ(min_value, min_value_d); +} + +static const std::vector DecimalAnyWidthValues = {-2, -1, 0, 1, 2}; + +TYPED_TEST(DecimalAnyWidthTest, ComparatorTest) { + for (size_t i = 0; i < DecimalAnyWidthValues.size(); i++) { + TypeParam d1(DecimalAnyWidthValues[i]); + for (size_t j = 0; j < DecimalAnyWidthValues.size(); j++) { + TypeParam d2(DecimalAnyWidthValues[j]); + + ASSERT_EQ(i == j, d1 == d2); + ASSERT_EQ(i != j, d1 != d2); + ASSERT_EQ(i < j, d1 < d2); + ASSERT_EQ(i <= j, d1 <= d2); + ASSERT_EQ(i > j, d1 > d2); + ASSERT_EQ(i >= j, d1 >= d2); + } + } +} + +TYPED_TEST(DecimalAnyWidthTest, UpCast) { + TypeParam d(42); + Decimal64 d64(d); + + ASSERT_EQ(d, d64); +} + +template +struct DecimalAnyWidthBinaryParams { + static const std::vector>> value; +}; + +template +const std::vector>> + DecimalAnyWidthBinaryParams::value = { + {"+", [](T x, T y) -> T { return x + y; }}, + {"-", [](T x, T y) -> T { return x - y; }}, + {"*", [](T x, T y) -> T { return x * y; }}, + {"/", [](T x, T y) -> T { return y == 0 ? 0 : x / y; }}, + {"%", [](T x, T y) -> T { return y == 0 ? 0 : x % y; }}, +}; + +TYPED_TEST(DecimalAnyWidthTest, BinaryOperations) { + using ValueType = typename arrow::DecimalAnyWidthTest_BinaryOperations_Test< + gtest_TypeParam_>::TypeParam::ValueType; + + auto DecimalFns = DecimalAnyWidthBinaryParams::value; + auto NumericFns = DecimalAnyWidthBinaryParams::value; + + for (size_t i = 0; i < DecimalFns.size(); i++) { + for (ValueType x : GetRandomNumbers(8)) { + for (ValueType y : GetRandomNumbers(8)) { + TypeParam d1(x), d2(y); + TypeParam result = DecimalFns[i].second(d1, d2); + ValueType reference = static_cast(NumericFns[i].second(x, y)); + ASSERT_EQ(reference, result) + << "(" << x << " " << DecimalFns[i].first << " " << y << " = " << reference + << ") != (" << d1 << " " << DecimalFns[i].first << " " << d2 << " = " + << result << ")"; + } + } + } +} + } // namespace arrow diff --git a/cpp/src/arrow/util/decimal_type_traits.h b/cpp/src/arrow/util/decimal_type_traits.h new file mode 100644 index 00000000000..e3202d5c3ca --- /dev/null +++ b/cpp/src/arrow/util/decimal_type_traits.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "arrow/type_fwd.h" + +namespace arrow { + +template +struct DecimalTypeTraits; + +#define DECIMAL_TYPE_TRAITS_DECL(width) \ + template <> \ + struct DecimalTypeTraits { \ + static constexpr Type::type Id = Type::DECIMAL##width; \ + using ArrayType = Decimal##width##Array; \ + using BuilderType = Decimal##width##Builder; \ + using ScalarType = Decimal##width##Scalar; \ + using TypeClass = Decimal##width##Type; \ + using ValueType = Decimal##width; \ + }; + +DECIMAL_TYPE_TRAITS_DECL(16) +DECIMAL_TYPE_TRAITS_DECL(32) +DECIMAL_TYPE_TRAITS_DECL(64) +DECIMAL_TYPE_TRAITS_DECL(128) +DECIMAL_TYPE_TRAITS_DECL(256) + +} // namespace arrow diff --git a/cpp/src/arrow/util/int_util_internal.h b/cpp/src/arrow/util/int_util_internal.h index de39229cfdd..246e25c15ea 100644 --- a/cpp/src/arrow/util/int_util_internal.h +++ b/cpp/src/arrow/util/int_util_internal.h @@ -79,6 +79,14 @@ SignedInt SafeSignedSubtract(SignedInt u, SignedInt v) { static_cast(v)); } +/// Signed multiply with well-defined behaviour on overflow (as unsigned) +template +SignedInt SafeSignedMultiply(SignedInt u, SignedInt v) { + using UnsignedInt = typename std::make_unsigned::type; + return static_cast(static_cast(u) * + static_cast(v)); +} + /// Signed left shift with well-defined behaviour on negative numbers or overflow template SignedInt SafeLeftShift(SignedInt u, Shift shift) { diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc index 851785081c7..70b2d3969d3 100644 --- a/cpp/src/arrow/visitor.cc +++ b/cpp/src/arrow/visitor.cc @@ -66,6 +66,9 @@ ARRAY_VISITOR_DEFAULT(StructArray) ARRAY_VISITOR_DEFAULT(SparseUnionArray) ARRAY_VISITOR_DEFAULT(DenseUnionArray) ARRAY_VISITOR_DEFAULT(DictionaryArray) +ARRAY_VISITOR_DEFAULT(Decimal16Array) +ARRAY_VISITOR_DEFAULT(Decimal32Array) +ARRAY_VISITOR_DEFAULT(Decimal64Array) ARRAY_VISITOR_DEFAULT(Decimal128Array) ARRAY_VISITOR_DEFAULT(Decimal256Array) ARRAY_VISITOR_DEFAULT(ExtensionArray) @@ -106,6 +109,9 @@ TYPE_VISITOR_DEFAULT(TimestampType) TYPE_VISITOR_DEFAULT(DayTimeIntervalType) TYPE_VISITOR_DEFAULT(MonthIntervalType) TYPE_VISITOR_DEFAULT(DurationType) +TYPE_VISITOR_DEFAULT(Decimal16Type) +TYPE_VISITOR_DEFAULT(Decimal32Type) +TYPE_VISITOR_DEFAULT(Decimal64Type) TYPE_VISITOR_DEFAULT(Decimal128Type) TYPE_VISITOR_DEFAULT(Decimal256Type) TYPE_VISITOR_DEFAULT(ListType) @@ -155,6 +161,9 @@ SCALAR_VISITOR_DEFAULT(TimestampScalar) SCALAR_VISITOR_DEFAULT(DayTimeIntervalScalar) SCALAR_VISITOR_DEFAULT(MonthIntervalScalar) SCALAR_VISITOR_DEFAULT(DurationScalar) +SCALAR_VISITOR_DEFAULT(Decimal16Scalar) +SCALAR_VISITOR_DEFAULT(Decimal32Scalar) +SCALAR_VISITOR_DEFAULT(Decimal64Scalar) SCALAR_VISITOR_DEFAULT(Decimal128Scalar) SCALAR_VISITOR_DEFAULT(Decimal256Scalar) SCALAR_VISITOR_DEFAULT(ListScalar) diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h index 0382e461199..c0281992d5c 100644 --- a/cpp/src/arrow/visitor.h +++ b/cpp/src/arrow/visitor.h @@ -53,6 +53,9 @@ class ARROW_EXPORT ArrayVisitor { virtual Status Visit(const DayTimeIntervalArray& array); virtual Status Visit(const MonthIntervalArray& array); virtual Status Visit(const DurationArray& array); + virtual Status Visit(const Decimal16Array& array); + virtual Status Visit(const Decimal32Array& array); + virtual Status Visit(const Decimal64Array& array); virtual Status Visit(const Decimal128Array& array); virtual Status Visit(const Decimal256Array& array); virtual Status Visit(const ListArray& array); @@ -96,6 +99,9 @@ class ARROW_EXPORT TypeVisitor { virtual Status Visit(const MonthIntervalType& type); virtual Status Visit(const DayTimeIntervalType& type); virtual Status Visit(const DurationType& type); + virtual Status Visit(const Decimal16Type& type); + virtual Status Visit(const Decimal32Type& type); + virtual Status Visit(const Decimal64Type& type); virtual Status Visit(const Decimal128Type& type); virtual Status Visit(const Decimal256Type& type); virtual Status Visit(const ListType& type); @@ -139,6 +145,9 @@ class ARROW_EXPORT ScalarVisitor { virtual Status Visit(const DayTimeIntervalScalar& scalar); virtual Status Visit(const MonthIntervalScalar& scalar); virtual Status Visit(const DurationScalar& scalar); + virtual Status Visit(const Decimal16Scalar& scalar); + virtual Status Visit(const Decimal32Scalar& scalar); + virtual Status Visit(const Decimal64Scalar& scalar); virtual Status Visit(const Decimal128Scalar& scalar); virtual Status Visit(const Decimal256Scalar& scalar); virtual Status Visit(const ListScalar& scalar); diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h index 132c35aeaa1..3e1f86f1585 100644 --- a/cpp/src/arrow/visitor_inline.h +++ b/cpp/src/arrow/visitor_inline.h @@ -67,6 +67,9 @@ namespace arrow { ACTION(Time64); \ ACTION(MonthInterval); \ ACTION(DayTimeInterval); \ + ACTION(Decimal16); \ + ACTION(Decimal32); \ + ACTION(Decimal64); \ ACTION(Decimal128); \ ACTION(Decimal256); \ ACTION(List); \ diff --git a/r/R/enums.R b/r/R/enums.R index 14910bc92e0..bf98ed88180 100644 --- a/r/R/enums.R +++ b/r/R/enums.R @@ -65,20 +65,23 @@ Type <- enum("Type::type", TIME64 = 20L, INTERVAL_MONTHS = 21L, INTERVAL_DAY_TIME = 22L, - DECIMAL = 23L, - DECIMAL256 = 24L, - LIST = 25L, - STRUCT = 26L, - SPARSE_UNION = 27L, - DENSE_UNION = 28L, - DICTIONARY = 29L, - MAP = 30L, - EXTENSION = 31L, - FIXED_SIZE_LIST = 32L, - DURATION = 33L, - LARGE_STRING = 34L, - LARGE_BINARY = 35L, - LARGE_LIST = 36L + DECIMAL16 = 23L, + DECIMAL32 = 24L, + DECIMAL64 = 25L, + DECIMAL = 26L, + DECIMAL256 = 27L, + LIST = 28L, + STRUCT = 29L, + SPARSE_UNION = 30L, + DENSE_UNION = 31L, + DICTIONARY = 32L, + MAP = 33L, + EXTENSION = 34L, + FIXED_SIZE_LIST = 35L, + DURATION = 36L, + LARGE_STRING = 37L, + LARGE_BINARY = 38L, + LARGE_LIST = 39L ) #' @rdname enums