From b064816367b85ddd51992cec4f4c4c3c70e0ed89 Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Thu, 14 Feb 2019 02:58:19 -0800 Subject: [PATCH 1/3] Introduce a new Duration type can that represent time deltas, and mark DayTimeInterval as deprecated. Provides implementation for Duration, DayTimeInterval and YearMonth interval in C++ and Java (for duration). Adds integration test. --- cpp/src/arrow/array.cc | 20 + cpp/src/arrow/array.h | 30 ++ cpp/src/arrow/array/builder_dict.cc | 6 + cpp/src/arrow/array/builder_primitive.h | 5 - cpp/src/arrow/array/builder_time.h | 70 +++ cpp/src/arrow/builder.cc | 15 + cpp/src/arrow/builder.h | 1 + cpp/src/arrow/compare.cc | 11 +- cpp/src/arrow/ipc/json-internal.cc | 157 +++++-- cpp/src/arrow/ipc/json-test.cc | 13 +- cpp/src/arrow/ipc/message.cc | 2 +- cpp/src/arrow/ipc/metadata-internal.cc | 47 +- cpp/src/arrow/ipc/read-write-test.cc | 3 +- cpp/src/arrow/ipc/test-common.cc | 25 + cpp/src/arrow/ipc/test-common.h | 3 + cpp/src/arrow/ipc/writer.cc | 3 + cpp/src/arrow/pretty_print.cc | 13 +- cpp/src/arrow/scalar.h | 21 + cpp/src/arrow/type-test.cc | 53 +++ cpp/src/arrow/type.cc | 19 + cpp/src/arrow/type.h | 91 +++- cpp/src/arrow/type_fwd.h | 17 +- cpp/src/arrow/type_traits.h | 40 +- cpp/src/arrow/visitor.cc | 12 +- cpp/src/arrow/visitor.h | 12 +- cpp/src/arrow/visitor_inline.h | 22 + format/Schema.fbs | 30 +- integration/integration_test.py | 85 +++- .../src/main/codegen/data/ArrowTypes.tdd | 5 + .../main/codegen/data/ValueVectorTypes.tdd | 4 + .../src/main/codegen/includes/vv_imports.ftl | 3 - .../codegen/templates/HolderReaderImpl.java | 2 + .../apache/arrow/vector/DurationVector.java | 426 ++++++++++++++++++ .../arrow/vector/IntervalDayVector.java | 29 +- .../arrow/vector/IntervalYearVector.java | 16 + .../org/apache/arrow/vector/TypeLayout.java | 6 + .../arrow/vector/ipc/JsonFileReader.java | 37 +- .../arrow/vector/ipc/JsonFileWriter.java | 20 +- .../org/apache/arrow/vector/types/Types.java | 25 + .../arrow/vector/TestDurationVector.java | 137 ++++++ .../arrow/vector/types/pojo/TestSchema.java | 24 +- 41 files changed, 1469 insertions(+), 91 deletions(-) create mode 100644 cpp/src/arrow/array/builder_time.h create mode 100644 java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java create mode 100644 java/vector/src/test/java/org/apache/arrow/vector/TestDurationVector.java diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc index b90945b020f..4897d0653dd 100644 --- a/cpp/src/arrow/array.cc +++ b/cpp/src/arrow/array.cc @@ -398,6 +398,26 @@ const uint8_t* FixedSizeBinaryArray::GetValue(int64_t i) const { return raw_values_ + (i + data_->offset) * byte_width_; } +// ---------------------------------------------------------------------- +// Day time interval + +DayTimeIntervalArray::DayTimeIntervalArray(const std::shared_ptr& data) { + SetData(data); +} + +DayTimeIntervalArray::DayTimeIntervalArray(const std::shared_ptr& type, + int64_t length, + const std::shared_ptr& data, + const std::shared_ptr& null_bitmap, + int64_t null_count, int64_t offset) + : PrimitiveArray(type, length, data, null_bitmap, null_count, offset) {} + +DayTimeIntervalType::DayMilliseconds DayTimeIntervalArray::GetValue(int64_t i) const { + DCHECK(i < length()); + return *reinterpret_cast( + raw_values_ + (i + data_->offset) * byte_width()); +} + // ---------------------------------------------------------------------- // Decimal diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h index 7a706ec8f47..0c6b28a4208 100644 --- a/cpp/src/arrow/array.h +++ b/cpp/src/arrow/array.h @@ -687,6 +687,36 @@ class ARROW_EXPORT FixedSizeBinaryArray : public PrimitiveArray { int32_t byte_width_; }; +/// DayTimeArray +/// --------------------- +/// \brief Array of Day and Millisecond values. +class ARROW_EXPORT DayTimeIntervalArray : public PrimitiveArray { + public: + using TypeClass = DayTimeIntervalType; + + explicit DayTimeIntervalArray(const std::shared_ptr& data); + + DayTimeIntervalArray(const std::shared_ptr& type, int64_t length, + const std::shared_ptr& data, + const std::shared_ptr& null_bitmap = NULLPTR, + int64_t null_count = kUnknownNullCount, int64_t offset = 0); + + TypeClass::DayMilliseconds GetValue(int64_t i) const; + TypeClass::DayMilliseconds Value(int64_t i) const { return GetValue(i); } + + // For compability with Take kernel. + TypeClass::DayMilliseconds GetView(int64_t i) const { return GetValue(i); } + + int32_t byte_width() const { return sizeof(TypeClass::DayMilliseconds); } + + const uint8_t* raw_values() const { return raw_values_ + data_->offset * byte_width(); } + + protected: + inline void SetData(const std::shared_ptr& data) { + this->PrimitiveArray::SetData(data); + } +}; + // ---------------------------------------------------------------------- // Decimal128Array diff --git a/cpp/src/arrow/array/builder_dict.cc b/cpp/src/arrow/array/builder_dict.cc index 8af36280d0d..72bfebfb532 100644 --- a/cpp/src/arrow/array/builder_dict.cc +++ b/cpp/src/arrow/array/builder_dict.cc @@ -54,6 +54,12 @@ struct UnifyDictionaryValues { " dictionaries is not implemented"); } + Status Visit(const DayTimeIntervalType&, void* = nullptr) { + return Status::NotImplemented( + "Unification of DayTime" + " dictionaries is not implemented"); + } + template Status Visit(const T&, typename internal::DictionaryTraits::MemoTableType* = nullptr) { diff --git a/cpp/src/arrow/array/builder_primitive.h b/cpp/src/arrow/array/builder_primitive.h index 4c7cbf2c25e..3d566846d19 100644 --- a/cpp/src/arrow/array/builder_primitive.h +++ b/cpp/src/arrow/array/builder_primitive.h @@ -258,11 +258,6 @@ using Int8Builder = NumericBuilder; using Int16Builder = NumericBuilder; using Int32Builder = NumericBuilder; using Int64Builder = NumericBuilder; -using TimestampBuilder = NumericBuilder; -using Time32Builder = NumericBuilder; -using Time64Builder = NumericBuilder; -using Date32Builder = NumericBuilder; -using Date64Builder = NumericBuilder; using HalfFloatBuilder = NumericBuilder; using FloatBuilder = NumericBuilder; diff --git a/cpp/src/arrow/array/builder_time.h b/cpp/src/arrow/array/builder_time.h new file mode 100644 index 00000000000..3ff783b1b1c --- /dev/null +++ b/cpp/src/arrow/array/builder_time.h @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Contains declarations of time related Arrow builder types. + +#pragma once + +#include + +#include "arrow/array.h" +#include "arrow/array/builder_base.h" +#include "arrow/array/builder_binary.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/buffer-builder.h" +#include "arrow/status.h" +#include "arrow/type_traits.h" +#include "arrow/util/macros.h" + +namespace arrow { + +class ARROW_EXPORT DayTimeIntervalBuilder : public ArrayBuilder { + public: + using DayMilliseconds = DayTimeIntervalType::DayMilliseconds; + + explicit DayTimeIntervalBuilder(MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT) + : DayTimeIntervalBuilder(day_time_interval(), pool) {} + + DayTimeIntervalBuilder(std::shared_ptr type, + MemoryPool* pool ARROW_MEMORY_POOL_DEFAULT) + : ArrayBuilder(type, pool), + builder_(fixed_size_binary(sizeof(DayMilliseconds)), pool) {} + + void Reset() override { builder_.Reset(); } + Status Resize(int64_t capacity) override { return builder_.Resize(capacity); } + Status Append(DayMilliseconds day_millis) { + return builder_.Append(reinterpret_cast(&day_millis)); + } + void UnsafeAppend(DayMilliseconds day_millis) { + builder_.UnsafeAppend(reinterpret_cast(&day_millis)); + } + using ArrayBuilder::UnsafeAppendNull; + Status AppendNull() override { return builder_.AppendNull(); } + Status AppendNulls(int64_t length) override { return builder_.AppendNulls(length); } + Status FinishInternal(std::shared_ptr* out) override { + auto result = builder_.FinishInternal(out); + if (*out != NULLPTR) { + (*out)->type = type(); + } + return result; + } + + private: + FixedSizeBinaryBuilder builder_; +}; + +} // namespace arrow diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index cb9160bee85..109ccabb611 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -95,6 +95,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, BUILDER_CASE(INT64, Int64Builder); BUILDER_CASE(DATE32, Date32Builder); BUILDER_CASE(DATE64, Date64Builder); + BUILDER_CASE(DURATION, DurationBuilder); BUILDER_CASE(TIME32, Time32Builder); BUILDER_CASE(TIME64, Time64Builder); BUILDER_CASE(TIMESTAMP, TimestampBuilder); @@ -111,6 +112,18 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, DictionaryBuilderCase visitor = {pool, dict_type, out}; return VisitTypeInline(*dict_type.dictionary()->type(), &visitor); } + case Type::INTERVAL: { + const auto& interval_type = internal::checked_cast(*type); + if (interval_type.interval_type() == IntervalType::MONTHS) { + out->reset(new MonthIntervalBuilder(type, pool)); + return Status::OK(); + } + if (interval_type.interval_type() == IntervalType::DAY_TIME) { + out->reset(new DayTimeIntervalBuilder(pool)); + return Status::OK(); + } + break; + } case Type::LIST: { std::unique_ptr value_builder; std::shared_ptr value_type = @@ -146,6 +159,8 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, type->ToString()); } } + return Status::NotImplemented("MakeBuilder: cannot construct builder for type ", + type->ToString()); } } // namespace arrow diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h index a7ab22c1bee..c0672c25a06 100644 --- a/cpp/src/arrow/builder.h +++ b/cpp/src/arrow/builder.h @@ -26,6 +26,7 @@ #include "arrow/array/builder_dict.h" // IWYU pragma: export #include "arrow/array/builder_nested.h" // IWYU pragma: export #include "arrow/array/builder_primitive.h" // IWYU pragma: export +#include "arrow/array/builder_time.h" // IWYU pragma: export #include "arrow/status.h" #include "arrow/util/visibility.h" diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc index 1b9b045101c..8fdd3cb0240 100644 --- a/cpp/src/arrow/compare.cc +++ b/cpp/src/arrow/compare.cc @@ -721,9 +721,18 @@ class TypeEqualsVisitor { return Status::OK(); } + template + typename std::enable_if::value, Status>::type Visit( + const T& left) { + const auto& right = checked_cast(right_); + result_ = right.interval_type() == left.interval_type(); + return Status::OK(); + } + template typename std::enable_if::value || - std::is_base_of::value, + std::is_base_of::value || + std::is_base_of::value, Status>::type Visit(const T& left) { const auto& right = checked_cast(right_); diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index dac4aba7ae7..bd64b914a47 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -43,6 +43,14 @@ #include "arrow/visitor_inline.h" namespace arrow { +namespace { +constexpr char kData[] = "DATA"; +constexpr char kDays[] = "days"; +constexpr char kDayTime[] = "DAY_TIME"; +constexpr char kDuration[] = "duration"; +constexpr char kMilliseconds[] = "milliseconds"; +constexpr char kYearMonth[] = "YEAR_MONTH"; +} // namespace class MemoryPool; @@ -203,12 +211,12 @@ class SchemaWriter { void WriteTypeMetadata(const IntervalType& type) { writer_->Key("unit"); - switch (type.unit()) { - case IntervalType::Unit::YEAR_MONTH: - writer_->String("YEAR_MONTH"); + switch (type.interval_type()) { + case IntervalType::MONTHS: + writer_->String(kYearMonth); break; - case IntervalType::Unit::DAY_TIME: - writer_->String("DAY_TIME"); + case IntervalType::DAY_TIME: + writer_->String(kDayTime); break; } } @@ -222,6 +230,11 @@ class SchemaWriter { } } + void WriteTypeMetadata(const DurationType& type) { + writer_->Key("unit"); + writer_->String(GetTimeUnitName(type.unit())); + } + void WriteTypeMetadata(const TimeType& type) { writer_->Key("unit"); writer_->String(GetTimeUnitName(type.unit())); @@ -327,7 +340,12 @@ class SchemaWriter { Status Visit(const Decimal128Type& type) { return WritePrimitive("decimal", type); } Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); } - Status Visit(const IntervalType& type) { return WritePrimitive("interval", type); } + Status Visit(const DurationType& type) { return WritePrimitive(kDuration, type); } + Status Visit(const MonthIntervalType& type) { return WritePrimitive("interval", type); } + + Status Visit(const DayTimeIntervalType& type) { + return WritePrimitive("interval", type); + } Status Visit(const ListType& type) { WriteName("list", type); @@ -459,6 +477,20 @@ class ArrayWriter { } } + void WriteDataValues(const DayTimeIntervalArray& arr) { + for (int64_t i = 0; i < arr.length(); ++i) { + writer_->StartObject(); + if (arr.IsValid(i)) { + const DayTimeIntervalType::DayMilliseconds dm = arr.GetValue(i); + writer_->Key(kDays); + writer_->Int(dm.days); + writer_->Key(kMilliseconds); + writer_->Int(dm.milliseconds); + } + writer_->EndObject(); + } + } + void WriteDataValues(const Decimal128Array& arr) { static const char null_string[] = "0"; for (int64_t i = 0; i < arr.length(); ++i) { @@ -483,7 +515,7 @@ class ArrayWriter { template void WriteDataField(const T& arr) { - writer_->Key("DATA"); + writer_->Key(kData); writer_->StartArray(); WriteDataValues(arr); writer_->EndArray(); @@ -763,24 +795,43 @@ static Status GetTime(const RjObject& json_type, std::shared_ptr* type return Status::OK(); } -static Status GetTimestamp(const RjObject& json_type, std::shared_ptr* type) { - const auto& it_unit = json_type.FindMember("unit"); - RETURN_NOT_STRING("unit", it_unit, json_type); - - std::string unit_str = it_unit->value.GetString(); - - TimeUnit::type unit; +static Status GetUnitFromString(const std::string& unit_str, TimeUnit::type* unit) { if (unit_str == "SECOND") { - unit = TimeUnit::SECOND; + *unit = TimeUnit::SECOND; } else if (unit_str == "MILLISECOND") { - unit = TimeUnit::MILLI; + *unit = TimeUnit::MILLI; } else if (unit_str == "MICROSECOND") { - unit = TimeUnit::MICRO; + *unit = TimeUnit::MICRO; } else if (unit_str == "NANOSECOND") { - unit = TimeUnit::NANO; + *unit = TimeUnit::NANO; } else { return Status::Invalid("Invalid time unit: ", unit_str); } + return Status::OK(); +} + +static Status GetDuration(const RjObject& json_type, std::shared_ptr* type) { + const auto& it_unit = json_type.FindMember("unit"); + RETURN_NOT_STRING("unit", it_unit, json_type); + + std::string unit_str = it_unit->value.GetString(); + + TimeUnit::type unit; + RETURN_NOT_OK(GetUnitFromString(unit_str, &unit)); + + *type = duration_type(unit); + + return Status::OK(); +} + +static Status GetTimestamp(const RjObject& json_type, std::shared_ptr* type) { + const auto& it_unit = json_type.FindMember("unit"); + RETURN_NOT_STRING("unit", it_unit, json_type); + + std::string unit_str = it_unit->value.GetString(); + + TimeUnit::type unit; + RETURN_NOT_OK(GetUnitFromString(unit_str, &unit)); const auto& it_tz = json_type.FindMember("timezone"); if (it_tz == json_type.MemberEnd()) { @@ -792,6 +843,22 @@ static Status GetTimestamp(const RjObject& json_type, std::shared_ptr* return Status::OK(); } +static Status GetInterval(const RjObject& json_type, std::shared_ptr* type) { + const auto& it_unit = json_type.FindMember("unit"); + RETURN_NOT_STRING("unit", it_unit, json_type); + + std::string unit_name = it_unit->value.GetString(); + + if (unit_name == kDayTime) { + *type = day_time_interval(); + } else if (unit_name == kYearMonth) { + *type = month_interval(); + } else { + return Status::Invalid("Invalid interval unit: " + unit_name); + } + return Status::OK(); +} + static Status GetUnion(const RjObject& json_type, const std::vector>& children, std::shared_ptr* type) { @@ -854,6 +921,10 @@ static Status GetType(const RjObject& json_type, return GetTime(json_type, type); } else if (type_name == "timestamp") { return GetTimestamp(json_type, type); + } else if (type_name == "interval") { + return GetInterval(json_type, type); + } else if (type_name == kDuration) { + return GetDuration(json_type, type); } else if (type_name == "list") { if (children.size() != 1) { return Status::Invalid("List must have exactly one child"); @@ -1015,13 +1086,15 @@ class ArrayReader { typename std::enable_if< std::is_base_of::value || std::is_base_of::value || std::is_base_of::value || - std::is_base_of::value || std::is_base_of::value, + std::is_base_of::value || std::is_base_of::value || + std::is_base_of::value || + std::is_base_of::value, Status>::type Visit(const T& type) { typename TypeTraits::BuilderType builder(type_, pool_); - const auto& json_data = obj_->FindMember("DATA"); - RETURN_NOT_ARRAY("DATA", json_data, *obj_); + const auto& json_data = obj_->FindMember(kData); + RETURN_NOT_ARRAY(kData, json_data, *obj_); const auto& json_data_arr = json_data->value.GetArray(); @@ -1044,8 +1117,8 @@ class ArrayReader { const T& type) { typename TypeTraits::BuilderType builder(pool_); - const auto& json_data = obj_->FindMember("DATA"); - RETURN_NOT_ARRAY("DATA", json_data, *obj_); + const auto& json_data = obj_->FindMember(kData); + RETURN_NOT_ARRAY(kData, json_data, *obj_); const auto& json_data_arr = json_data->value.GetArray(); @@ -1082,6 +1155,33 @@ class ArrayReader { return builder.Finish(&result_); } + Status Visit(const DayTimeIntervalType& type) { + DayTimeIntervalBuilder builder(pool_); + + const auto& json_data = obj_->FindMember(kData); + RETURN_NOT_ARRAY(kData, json_data, *obj_); + + const auto& json_data_arr = json_data->value.GetArray(); + + DCHECK_EQ(static_cast(json_data_arr.Size()), length_) + << "data length: " << json_data_arr.Size() << " != length_: " << length_; + + for (int i = 0; i < length_; ++i) { + if (!is_valid_[i]) { + RETURN_NOT_OK(builder.AppendNull()); + continue; + } + + const rj::Value& val = json_data_arr[i]; + DCHECK(val.IsObject()); + DayTimeIntervalType::DayMilliseconds dm = {0, 0}; + dm.days = val[kDays].GetInt(); + dm.milliseconds = val[kMilliseconds].GetInt(); + RETURN_NOT_OK(builder.Append(dm)); + } + return builder.Finish(&result_); + } + template typename std::enable_if::value && !std::is_base_of::value, @@ -1089,8 +1189,8 @@ class ArrayReader { Visit(const T& type) { typename TypeTraits::BuilderType builder(type_, pool_); - const auto& json_data = obj_->FindMember("DATA"); - RETURN_NOT_ARRAY("DATA", json_data, *obj_); + const auto& json_data = obj_->FindMember(kData); + RETURN_NOT_ARRAY(kData, json_data, *obj_); const auto& json_data_arr = json_data->value.GetArray(); @@ -1130,8 +1230,8 @@ class ArrayReader { const T& type) { typename TypeTraits::BuilderType builder(type_, pool_); - const auto& json_data = obj_->FindMember("DATA"); - RETURN_NOT_ARRAY("DATA", json_data, *obj_); + const auto& json_data = obj_->FindMember(kData); + RETURN_NOT_ARRAY(kData, json_data, *obj_); const auto& json_data_arr = json_data->value.GetArray(); @@ -1487,7 +1587,8 @@ Status WriteRecordBatch(const RecordBatch& batch, RjWriter* writer) { const std::shared_ptr& column = batch.column(i); DCHECK_EQ(batch.num_rows(), column->length()) - << "Array length did not match record batch length"; + << "Array length did not match record batch length: " << batch.num_rows() + << " != " << column->length() << " " << batch.column_name(i); RETURN_NOT_OK(WriteArray(batch.column_name(i), *column, writer)); } diff --git a/cpp/src/arrow/ipc/json-test.cc b/cpp/src/arrow/ipc/json-test.cc index fdbf951381a..df87671619c 100644 --- a/cpp/src/arrow/ipc/json-test.cc +++ b/cpp/src/arrow/ipc/json-test.cc @@ -370,12 +370,13 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) { ASSERT_TRUE(batch->column(1)->Equals(bar)); } -#define BATCH_CASES() \ - ::testing::Values( \ - &MakeIntRecordBatch, &MakeListRecordBatch, &MakeFixedSizeListRecordBatch, \ - &MakeNonNullRecordBatch, &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, \ - &MakeStringTypesRecordBatchWithNulls, &MakeStruct, &MakeUnion, &MakeDates, \ - &MakeTimestamps, &MakeTimes, &MakeFWBinary, &MakeDecimal, &MakeDictionary); +#define BATCH_CASES() \ + ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, \ + &MakeFixedSizeListRecordBatch, &MakeNonNullRecordBatch, \ + &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, \ + &MakeStringTypesRecordBatchWithNulls, &MakeStruct, &MakeUnion, \ + &MakeDates, &MakeTimestamps, &MakeTimes, &MakeFWBinary, \ + &MakeDecimal, &MakeDictionary, &MakeIntervals); class TestJsonRoundTrip : public ::testing::TestWithParam { public: diff --git a/cpp/src/arrow/ipc/message.cc b/cpp/src/arrow/ipc/message.cc index 23709a46192..f6d43e12a62 100644 --- a/cpp/src/arrow/ipc/message.cc +++ b/cpp/src/arrow/ipc/message.cc @@ -144,7 +144,7 @@ bool Message::Equals(const Message& other) const { Status Message::ReadFrom(const std::shared_ptr& metadata, io::InputStream* stream, std::unique_ptr* out) { auto data = metadata->data(); - flatbuffers::Verifier verifier(data, metadata->size(), 128); + flatbuffers::Verifier verifier(data, metadata->size(), /*max_depth=*/128); if (!flatbuf::VerifyMessageBuffer(verifier)) { return Status::IOError("Invalid flatbuffers message."); } diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc index 65d86418fae..46b99b12554 100644 --- a/cpp/src/arrow/ipc/metadata-internal.cc +++ b/cpp/src/arrow/ipc/metadata-internal.cc @@ -285,8 +285,28 @@ static Status ConcreteTypeFromFlatbuffer( } return Status::OK(); } - case flatbuf::Type_Interval: - return Status::NotImplemented("Interval"); + case flatbuf::Type_Duration: { + auto duration = static_cast(type_data); + TimeUnit::type unit = FromFlatbufferUnit(duration->unit()); + *out = duration_type(unit); + return Status::OK(); + } + + case flatbuf::Type_Interval: { + auto i_type = static_cast(type_data); + switch (i_type->unit()) { + case flatbuf::IntervalUnit_YEAR_MONTH: { + *out = month_interval(); + return Status::OK(); + } + case flatbuf::IntervalUnit_DAY_TIME: { + *out = day_time_interval(); + return Status::OK(); + } + } + return Status::NotImplemented("Unrecognized interval type."); + } + case flatbuf::Type_List: if (children.size() != 1) { return Status::Invalid("List must have exactly 1 child field"); @@ -308,7 +328,8 @@ static Status ConcreteTypeFromFlatbuffer( return UnionFromFlatbuffer(static_cast(type_data), children, out); default: - return Status::Invalid("Unrecognized type"); + return Status::Invalid("Unrecognized type:" + + std::to_string(static_cast(type))); } } @@ -538,6 +559,26 @@ class FieldToFlatbufferVisitor { return Status::OK(); } + Status Visit(const DurationType& type) { + fb_type_ = flatbuf::Type_Duration; + flatbuf::TimeUnit fb_unit = ToFlatbufferUnit(type.unit()); + type_offset_ = flatbuf::CreateDuration(fbb_, fb_unit).Union(); + return Status::OK(); + } + + Status Visit(const DayTimeIntervalType& type) { + fb_type_ = flatbuf::Type_Interval; + type_offset_ = flatbuf::CreateInterval(fbb_, flatbuf::IntervalUnit_DAY_TIME).Union(); + return Status::OK(); + } + + Status Visit(const MonthIntervalType& type) { + fb_type_ = flatbuf::Type_Interval; + type_offset_ = + flatbuf::CreateInterval(fbb_, flatbuf::IntervalUnit_YEAR_MONTH).Union(); + return Status::OK(); + } + Status Visit(const DecimalType& type) { const auto& dec_type = checked_cast(type); fb_type_ = flatbuf::Type_Decimal; diff --git a/cpp/src/arrow/ipc/read-write-test.cc b/cpp/src/arrow/ipc/read-write-test.cc index 0408a1712fe..5f76545e839 100644 --- a/cpp/src/arrow/ipc/read-write-test.cc +++ b/cpp/src/arrow/ipc/read-write-test.cc @@ -214,7 +214,8 @@ TEST_F(TestSchemaMetadata, KeyValueMetadata) { &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, \ &MakeStringTypesRecordBatchWithNulls, &MakeStruct, &MakeUnion, \ &MakeDictionary, &MakeDates, &MakeTimestamps, &MakeTimes, \ - &MakeFWBinary, &MakeNull, &MakeDecimal, &MakeBooleanBatch); + &MakeFWBinary, &MakeNull, &MakeDecimal, &MakeBooleanBatch, \ + &MakeIntervals); static int g_file_number = 0; diff --git a/cpp/src/arrow/ipc/test-common.cc b/cpp/src/arrow/ipc/test-common.cc index 354aba7caed..555d7ca7275 100644 --- a/cpp/src/arrow/ipc/test-common.cc +++ b/cpp/src/arrow/ipc/test-common.cc @@ -580,6 +580,31 @@ Status MakeTimestamps(std::shared_ptr* out) { return Status::OK(); } +Status MakeIntervals(std::shared_ptr* out) { + std::vector is_valid = {true, true, true, false, true, true, true}; + auto f0 = field("f0", duration_type(TimeUnit::MILLI)); + auto f1 = field("f1", duration_type(TimeUnit::NANO)); + auto f2 = field("f2", duration_type(TimeUnit::SECOND)); + auto f3 = field("f3", day_time_interval()); + auto f4 = field("f4", month_interval()); + auto schema = ::arrow::schema({f0, f1, f2, f3, f4}); + + std::vector ts_values = {1489269000000, 1489270000000, 1489271000000, + 1489272000000, 1489272000000, 1489273000000}; + + std::shared_ptr a0, a1, a2, a3, a4; + ArrayFromVector(f0->type(), is_valid, ts_values, &a0); + ArrayFromVector(f1->type(), is_valid, ts_values, &a1); + ArrayFromVector(f2->type(), is_valid, ts_values, &a2); + ArrayFromVector( + f3->type(), is_valid, {{0, 0}, {0, 1}, {1, 1}, {2, 1}, {3, 4}, {-1, -1}}, &a3); + ArrayFromVector(f4->type(), is_valid, {0, -1, 1, 2, -2, 24}, + &a4); + + *out = RecordBatch::Make(schema, a0->length(), {a0, a1, a2, a3, a4}); + return Status::OK(); +} + Status MakeTimes(std::shared_ptr* out) { std::vector is_valid = {true, true, true, false, true, true, true}; auto f0 = field("f0", time32(TimeUnit::MILLI)); diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h index c49d21bdf5b..adbc57bfe26 100644 --- a/cpp/src/arrow/ipc/test-common.h +++ b/cpp/src/arrow/ipc/test-common.h @@ -112,6 +112,9 @@ Status MakeDates(std::shared_ptr* out); ARROW_EXPORT Status MakeTimestamps(std::shared_ptr* out); +ARROW_EXPORT +Status MakeIntervals(std::shared_ptr* out); + ARROW_EXPORT Status MakeTimes(std::shared_ptr* out); diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc index bc89dc48da4..4089cdd2a9e 100644 --- a/cpp/src/arrow/ipc/writer.cc +++ b/cpp/src/arrow/ipc/writer.cc @@ -304,6 +304,9 @@ class RecordBatchSerializer : public ArrayVisitor { VISIT_FIXED_WIDTH(Date32Array) VISIT_FIXED_WIDTH(Date64Array) VISIT_FIXED_WIDTH(TimestampArray) + VISIT_FIXED_WIDTH(DurationArray) + VISIT_FIXED_WIDTH(MonthIntervalArray) + VISIT_FIXED_WIDTH(DayTimeIntervalArray) VISIT_FIXED_WIDTH(Time32Array) VISIT_FIXED_WIDTH(Time64Array) VISIT_FIXED_WIDTH(FixedSizeBinaryArray) diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc index 1aa2279e3c3..97721b1a41b 100644 --- a/cpp/src/arrow/pretty_print.cc +++ b/cpp/src/arrow/pretty_print.cc @@ -182,6 +182,17 @@ class ArrayPrinter : public PrettyPrinter { return Status::OK(); } + template + inline + typename std::enable_if::value, Status>::type + WriteDataValues(const T& array) { + WriteValues(array, [&](int64_t i) { + auto day_millis = array.GetValue(i); + (*sink_) << day_millis.days << "d" << day_millis.milliseconds << "ms"; + }); + return Status::OK(); + } + template inline typename std::enable_if::value, Status>::type WriteDataValues( const T& array) { @@ -279,8 +290,6 @@ class ArrayPrinter : public PrettyPrinter { return Status::OK(); } - Status Visit(const IntervalArray&) { return Status::NotImplemented("interval"); } - Status Visit(const ExtensionArray& array) { return Print(*array.storage()); } Status WriteValidityBitmap(const Array& array); diff --git a/cpp/src/arrow/scalar.h b/cpp/src/arrow/scalar.h index c6609d04af4..51b5e71c345 100644 --- a/cpp/src/arrow/scalar.h +++ b/cpp/src/arrow/scalar.h @@ -143,6 +143,27 @@ class ARROW_EXPORT TimestampScalar : public internal::PrimitiveScalar { bool is_valid = true); }; +class ARROW_EXPORT DurationScalar : public internal::PrimitiveScalar { + public: + int64_t value; + DurationScalar(int64_t value, const std::shared_ptr& type, + bool is_valid = true); +}; + +class ARROW_EXPORT MonthIntervalScalar : public internal::PrimitiveScalar { + public: + int32_t value; + MonthIntervalScalar(int32_t value, const std::shared_ptr& type, + bool is_valid = true); +}; + +class ARROW_EXPORT DayTimeIntervalScalar : public Scalar { + public: + DayTimeIntervalType::DayMilliseconds value; + DayTimeIntervalScalar(DayTimeIntervalType::DayMilliseconds value, + const std::shared_ptr& type, bool is_valid = true); +}; + struct ARROW_EXPORT Decimal128Scalar : public Scalar { Decimal128 value; Decimal128Scalar(const Decimal128& value, const std::shared_ptr& type, diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc index aa2419effb7..ad6667a0b57 100644 --- a/cpp/src/arrow/type-test.cc +++ b/cpp/src/arrow/type-test.cc @@ -431,6 +431,59 @@ TEST(TestTimeType, ToString) { ASSERT_EQ("time64[us]", t4->ToString()); } +TEST(TestMonthIntervalType, Equals) { + MonthIntervalType t1; + MonthIntervalType t2; + DayTimeIntervalType t3; + + ASSERT_TRUE(t1.Equals(t2)); + ASSERT_FALSE(t1.Equals(t3)); +} + +TEST(TestMonthIntervalType, ToString) { + auto t1 = month_interval(); + + ASSERT_EQ("month_interval", t1->ToString()); +} + +TEST(TestDayTimeIntervalType, Equals) { + DayTimeIntervalType t1; + DayTimeIntervalType t2; + MonthIntervalType t3; + + ASSERT_TRUE(t1.Equals(t2)); + ASSERT_FALSE(t1.Equals(t3)); +} + +TEST(TestDayTimeIntervalType, ToString) { + auto t1 = day_time_interval(); + + ASSERT_EQ("day_time_interval", t1->ToString()); +} + +TEST(TestDurationType, Equals) { + DurationType t1; + DurationType t2; + DurationType t3(TimeUnit::NANO); + DurationType t4(TimeUnit::NANO); + + ASSERT_TRUE(t1.Equals(t2)); + ASSERT_FALSE(t1.Equals(t3)); + ASSERT_TRUE(t3.Equals(t4)); +} + +TEST(TestDurationType, ToString) { + auto t1 = duration_type(TimeUnit::MILLI); + auto t2 = duration_type(TimeUnit::NANO); + auto t3 = duration_type(TimeUnit::SECOND); + auto t4 = duration_type(TimeUnit::MICRO); + + ASSERT_EQ("duration[ms]", t1->ToString()); + ASSERT_EQ("duration[ns]", t2->ToString()); + ASSERT_EQ("duration[s]", t3->ToString()); + ASSERT_EQ("duration[us]", t4->ToString()); +} + TEST(TestTimestampType, Equals) { TimestampType t1; TimestampType t2; diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index 3fd8e543e1d..9ef6a9bddd9 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -200,6 +200,13 @@ std::string TimestampType::ToString() const { return ss.str(); } +// Duration types +std::string DurationType::ToString() const { + std::stringstream ss; + ss << "duration[" << this->unit_ << "]"; + return ss.str(); +} + // ---------------------------------------------------------------------- // Union type @@ -547,6 +554,18 @@ std::shared_ptr fixed_size_binary(int32_t byte_width) { return std::make_shared(byte_width); } +std::shared_ptr duration_type(TimeUnit::type unit) { + return std::make_shared(unit); +} + +std::shared_ptr day_time_interval() { + return std::make_shared(); +} + +std::shared_ptr month_interval() { + return std::make_shared(); +} + std::shared_ptr timestamp(TimeUnit::type unit) { return std::make_shared(unit); } diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index f6ec81b49eb..fcd86378d30 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -139,7 +139,11 @@ struct Type { EXTENSION, /// Fixed size list of some logical type - FIXED_SIZE_LIST + FIXED_SIZE_LIST, + + /// Measure of elapsed time in either seconds, milliseconds, microseconds + /// or nanoseconds. + DURATION }; }; @@ -756,25 +760,81 @@ class ARROW_EXPORT TimestampType : public FixedWidthType, public ParametricType std::string timezone_; }; -class ARROW_EXPORT IntervalType : public FixedWidthType { +// Holds different types of intervals. +class ARROW_EXPORT IntervalType : public FixedWidthType, public ParametricType { public: - enum class Unit : char { YEAR_MONTH = 0, DAY_TIME = 1 }; + enum type { MONTHS, DAY_TIME }; + IntervalType() : FixedWidthType(Type::INTERVAL) {} - using c_type = int64_t; + virtual type interval_type() const = 0; + virtual ~IntervalType() = default; +}; + +/// \brief Represents a some number of months. +/// +/// Type representing a number of months. Corresponeds to YearMonth type +/// in Schema.fbs (Years are defined as 12 months). +class ARROW_EXPORT MonthIntervalType : public IntervalType { + public: + using c_type = int32_t; static constexpr Type::type type_id = Type::INTERVAL; - int bit_width() const override { return static_cast(sizeof(int64_t) * CHAR_BIT); } + IntervalType::type interval_type() const override { return IntervalType::MONTHS; } + + int bit_width() const override { return static_cast(sizeof(c_type) * CHAR_BIT); } - explicit IntervalType(Unit unit = Unit::YEAR_MONTH) - : FixedWidthType(Type::INTERVAL), unit_(unit) {} + MonthIntervalType() {} std::string ToString() const override { return name(); } - std::string name() const override { return "interval"; } + std::string name() const override { return "month_interval"; } +}; - Unit unit() const { return unit_; } +/// \brief Represents a number of days and milliseconds (fraction of day). +class ARROW_EXPORT DayTimeIntervalType : public IntervalType { + public: + struct DayMilliseconds { + int32_t days; + int32_t milliseconds; + bool operator==(DayMilliseconds other) { + return this->days == other.days && this->milliseconds == other.milliseconds; + } + bool operator!=(DayMilliseconds other) { return !(*this == other); } + }; + using c_type = DayMilliseconds; + static_assert(sizeof(DayMilliseconds) == 8, + "DayMilliseconds struct assumed to be of size 8 bytes"); + static constexpr Type::type type_id = Type::INTERVAL; + IntervalType::type interval_type() const override { return IntervalType::DAY_TIME; } + + DayTimeIntervalType() {} + + int bit_width() const override { return static_cast(sizeof(c_type) * CHAR_BIT); } + + std::string ToString() const override { return name(); } + std::string name() const override { return "day_time_interval"; } +}; + +// \brief Represents an amount of elapsed time without any relation to a calendar +// artifact. +class ARROW_EXPORT DurationType : public FixedWidthType, public ParametricType { + public: + using Unit = TimeUnit; + + static constexpr Type::type type_id = Type::DURATION; + using c_type = int64_t; + + int bit_width() const override { return static_cast(sizeof(int64_t) * CHAR_BIT); } + + explicit DurationType(TimeUnit::type unit = TimeUnit::MILLI) + : FixedWidthType(Type::DURATION), unit_(unit) {} + + std::string ToString() const override; + std::string name() const override { return "duration"; } + + TimeUnit::type unit() const { return unit_; } private: - Unit unit_; + TimeUnit::type unit_; }; // ---------------------------------------------------------------------- @@ -906,7 +966,7 @@ class ARROW_EXPORT Schema { /// \addtogroup type-factories /// @{ -/// \brief Create a FixedSizeBinaryType instance +/// \brief Create a FixedSizeBinaryType instance. ARROW_EXPORT std::shared_ptr fixed_size_binary(int32_t byte_width); @@ -931,6 +991,15 @@ std::shared_ptr fixed_size_list(const std::shared_ptr& value_ty ARROW_EXPORT std::shared_ptr fixed_size_list(const std::shared_ptr& value_type, int32_t list_size); +/// \brief Return an Duration instance (naming use _type to avoid namespace conflict with +/// built in time clases). +std::shared_ptr ARROW_EXPORT duration_type(TimeUnit::type unit); + +/// \brief Return an DayTimeIntervalType instance +std::shared_ptr ARROW_EXPORT day_time_interval(); + +/// \brief Return an MonthIntervalType instance +std::shared_ptr ARROW_EXPORT month_interval(); /// \brief Create a TimestampType instance from its unit ARROW_EXPORT diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h index d6f5a0ffa13..3211d2a6ff6 100644 --- a/cpp/src/arrow/type_fwd.h +++ b/cpp/src/arrow/type_fwd.h @@ -154,9 +154,20 @@ using TimestampArray = NumericArray; using TimestampBuilder = NumericBuilder; class TimestampScalar; -class IntervalType; -using IntervalArray = NumericArray; -class IntervalScalar; +class MonthIntervalType; +using MonthIntervalArray = NumericArray; +using MonthIntervalBuilder = NumericBuilder; +class MonthIntervalScalar; + +class DayTimeIntervalType; +class DayTimeIntervalArray; +class DayTimeIntervalBuilder; +class DayTimeIntervalScalar; + +class DurationType; +using DurationArray = NumericArray; +using DurationBuilder = NumericBuilder; +class DurationScalar; class ExtensionType; class ExtensionArray; diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h index 3fd5ea143fe..13612ba2ea5 100644 --- a/cpp/src/arrow/type_traits.h +++ b/cpp/src/arrow/type_traits.h @@ -150,6 +150,42 @@ struct TypeTraits { constexpr static bool is_parameter_free = false; }; +template <> +struct TypeTraits { + using ArrayType = DurationArray; + using BuilderType = DurationBuilder; + using ScalarType = DurationScalar; + + static constexpr int64_t bytes_required(int64_t elements) { + return elements * static_cast(sizeof(int64_t)); + } + constexpr static bool is_parameter_free = false; +}; + +template <> +struct TypeTraits { + using ArrayType = DayTimeIntervalArray; + using BuilderType = DayTimeIntervalBuilder; + using ScalarType = DayTimeIntervalScalar; + + static constexpr int64_t bytes_required(int64_t elements) { + return elements * static_cast(sizeof(DayTimeIntervalType::DayMilliseconds)); + } + constexpr static bool is_parameter_free = true; +}; + +template <> +struct TypeTraits { + using ArrayType = MonthIntervalArray; + using BuilderType = MonthIntervalBuilder; + using ScalarType = MonthIntervalScalar; + + static constexpr int64_t bytes_required(int64_t elements) { + return elements * static_cast(sizeof(int32_t)); + } + constexpr static bool is_parameter_free = true; +}; + template <> struct TypeTraits { using ArrayType = Time32Array; @@ -298,7 +334,9 @@ template struct has_c_type { static constexpr bool value = (std::is_base_of::value || std::is_base_of::value || - std::is_base_of::value || std::is_base_of::value); + std::is_base_of::value || std::is_base_of::value || + std::is_base_of::value || + std::is_base_of::value); }; template diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc index 1fd4bcf21a1..9f28b1516ec 100644 --- a/cpp/src/arrow/visitor.cc +++ b/cpp/src/arrow/visitor.cc @@ -53,7 +53,9 @@ ARRAY_VISITOR_DEFAULT(Date64Array) ARRAY_VISITOR_DEFAULT(Time32Array) ARRAY_VISITOR_DEFAULT(Time64Array) ARRAY_VISITOR_DEFAULT(TimestampArray) -ARRAY_VISITOR_DEFAULT(IntervalArray) +ARRAY_VISITOR_DEFAULT(DayTimeIntervalArray) +ARRAY_VISITOR_DEFAULT(MonthIntervalArray) +ARRAY_VISITOR_DEFAULT(DurationArray) ARRAY_VISITOR_DEFAULT(ListArray) ARRAY_VISITOR_DEFAULT(FixedSizeListArray) ARRAY_VISITOR_DEFAULT(StructArray) @@ -93,7 +95,9 @@ TYPE_VISITOR_DEFAULT(Date32Type) TYPE_VISITOR_DEFAULT(Time32Type) TYPE_VISITOR_DEFAULT(Time64Type) TYPE_VISITOR_DEFAULT(TimestampType) -TYPE_VISITOR_DEFAULT(IntervalType) +TYPE_VISITOR_DEFAULT(DayTimeIntervalType) +TYPE_VISITOR_DEFAULT(MonthIntervalType) +TYPE_VISITOR_DEFAULT(DurationType) TYPE_VISITOR_DEFAULT(Decimal128Type) TYPE_VISITOR_DEFAULT(ListType) TYPE_VISITOR_DEFAULT(FixedSizeListType) @@ -134,7 +138,9 @@ SCALAR_VISITOR_DEFAULT(Date32Scalar) SCALAR_VISITOR_DEFAULT(Time32Scalar) SCALAR_VISITOR_DEFAULT(Time64Scalar) SCALAR_VISITOR_DEFAULT(TimestampScalar) -SCALAR_VISITOR_DEFAULT(IntervalScalar) +SCALAR_VISITOR_DEFAULT(DayTimeIntervalScalar) +SCALAR_VISITOR_DEFAULT(MonthIntervalScalar) +SCALAR_VISITOR_DEFAULT(DurationScalar) SCALAR_VISITOR_DEFAULT(Decimal128Scalar) SCALAR_VISITOR_DEFAULT(ListScalar) SCALAR_VISITOR_DEFAULT(FixedSizeListScalar) diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h index 41f7dc6aeea..1b40ce4efba 100644 --- a/cpp/src/arrow/visitor.h +++ b/cpp/src/arrow/visitor.h @@ -49,7 +49,9 @@ class ARROW_EXPORT ArrayVisitor { virtual Status Visit(const Time32Array& array); virtual Status Visit(const Time64Array& array); virtual Status Visit(const TimestampArray& array); - virtual Status Visit(const IntervalArray& array); + virtual Status Visit(const DayTimeIntervalArray& array); + virtual Status Visit(const MonthIntervalArray& array); + virtual Status Visit(const DurationArray& array); virtual Status Visit(const Decimal128Array& array); virtual Status Visit(const ListArray& array); virtual Status Visit(const FixedSizeListArray& array); @@ -84,7 +86,9 @@ class ARROW_EXPORT TypeVisitor { virtual Status Visit(const Time32Type& type); virtual Status Visit(const Time64Type& type); virtual Status Visit(const TimestampType& type); - virtual Status Visit(const IntervalType& type); + virtual Status Visit(const MonthIntervalType& type); + virtual Status Visit(const DayTimeIntervalType& type); + virtual Status Visit(const DurationType& type); virtual Status Visit(const Decimal128Type& type); virtual Status Visit(const ListType& type); virtual Status Visit(const FixedSizeListType& type); @@ -119,7 +123,9 @@ class ARROW_EXPORT ScalarVisitor { virtual Status Visit(const Time32Scalar& scalar); virtual Status Visit(const Time64Scalar& scalar); virtual Status Visit(const TimestampScalar& scalar); - virtual Status Visit(const IntervalScalar& scalar); + virtual Status Visit(const DayTimeIntervalScalar& scalar); + virtual Status Visit(const MonthIntervalScalar& scalar); + virtual Status Visit(const DurationScalar& scalar); virtual Status Visit(const Decimal128Scalar& scalar); virtual Status Visit(const ListScalar& scalar); virtual Status Visit(const FixedSizeListScalar& scalar); diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h index 414a227c803..4699238690b 100644 --- a/cpp/src/arrow/visitor_inline.h +++ b/cpp/src/arrow/visitor_inline.h @@ -49,6 +49,7 @@ namespace arrow { ACTION(String); \ ACTION(Binary); \ ACTION(FixedSizeBinary); \ + ACTION(Duration); \ ACTION(Date32); \ ACTION(Date64); \ ACTION(Timestamp); \ @@ -70,6 +71,16 @@ template inline Status VisitTypeInline(const DataType& type, VISITOR* visitor) { switch (type.id()) { ARROW_GENERATE_FOR_ALL_TYPES(TYPE_VISIT_INLINE); + case Type::INTERVAL: { + const auto& interval_type = dynamic_cast(type); + if (interval_type.interval_type() == IntervalType::MONTHS) { + return visitor->Visit(internal::checked_cast(type)); + } + if (interval_type.interval_type() == IntervalType::DAY_TIME) { + return visitor->Visit(internal::checked_cast(type)); + } + break; + } default: break; } @@ -88,6 +99,17 @@ template inline Status VisitArrayInline(const Array& array, VISITOR* visitor) { switch (array.type_id()) { ARROW_GENERATE_FOR_ALL_TYPES(ARRAY_VISIT_INLINE); + case Type::INTERVAL: { + const auto& interval_type = dynamic_cast(*array.type()); + if (interval_type.interval_type() == IntervalType::MONTHS) { + return visitor->Visit(internal::checked_cast(array)); + } + if (interval_type.interval_type() == IntervalType::DAY_TIME) { + return visitor->Visit(internal::checked_cast(array)); + } + break; + } + default: break; } diff --git a/format/Schema.fbs b/format/Schema.fbs index 9e52a8d5f9e..0b3139042e8 100644 --- a/format/Schema.fbs +++ b/format/Schema.fbs @@ -30,7 +30,7 @@ enum MetadataVersion:short { V3, /// >= 0.8.0 - V4 + V4, } /// These are stored in the flatbuffer in the Type union below @@ -183,10 +183,35 @@ table Timestamp { } enum IntervalUnit: short { YEAR_MONTH, DAY_TIME} +// A "calendar" interval which models types that don't necessarily +// have a precise duration without the context of a base timestamp (e.g. +// days can differ in length during day light savings time transitions). +// YEAR_MONTH - Indicates the number of elapsed whole months, stored as +// 4-byte integers. +// DAY_TIME - Indicates the number of elapsed days and milliseconds, +// stored as 2 contiguous 32-bit integers (8-bytes in total). Support +// of this IntervalUnit is not required for full arrow compatibility. table Interval { unit: IntervalUnit; } +// An absolute length of time unrelated to any calendar artifacts. +// +// For the purposes of Arrow Implementations, adding this value to a Timestamp +// ("t1") naively (i.e. simply summing the two number) is acceptable even +// though in some cases the resulting Timestamp (t2) would not account for +// leap-seconds during the elapsed time between "t1" and "t2". Similarly, +// representing the difference between two Unix timestamp is acceptable, but +// would yield a value that is possibly a few seconds off from the true elapsed +// time. +// +// The resolution defaults to millisecond, but can be any of the other +// supported TimeUnit values as with Timestamp and Time types. This type is +// always represented as an 8-byte integer. +table Duration { + unit: TimeUnit = MILLISECOND; +} + /// ---------------------------------------------------------------------- /// Top-level Type value, enabling extensible type-specific metadata. We can /// add new logical types to Type without breaking backwards compatibility @@ -208,7 +233,8 @@ union Type { Union, FixedSizeBinary, FixedSizeList, - Map + Map, + Duration, } /// ---------------------------------------------------------------------- diff --git a/integration/integration_test.py b/integration/integration_test.py index b184cd2cdb3..4c3a354d31d 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -328,6 +328,68 @@ def _get_type(self): return OrderedDict(fields) +class DurationIntervalType(IntegerType): + def __init__(self, name, unit='s', nullable=True): + min_val, max_val = np.iinfo('int64').min, np.iinfo('int64').max, + super(DurationIntervalType, self).__init__( + name, True, 64, nullable=nullable, + min_value=min_val, + max_value=max_val) + self.unit = unit + + def _get_type(self): + fields = [ + ('name', 'duration'), + ('unit', TIMEUNIT_NAMES[self.unit]) + ] + + return OrderedDict(fields) + + +class YearMonthIntervalType(IntegerType): + def __init__(self, name, nullable=True): + min_val, max_val = [-10000*12, 10000*12] # +/- 10000 years. + super(YearMonthIntervalType, self).__init__( + name, True, 32, nullable=nullable, + min_value=min_val, + max_value=max_val) + + def _get_type(self): + fields = [ + ('name', 'interval'), + ('unit', 'YEAR_MONTH'), + ] + + return OrderedDict(fields) + + +class DayTimeIntervalType(PrimitiveType): + def __init__(self, name, nullable=True): + super(DayTimeIntervalType, self).__init__(name, nullable=True) + + @property + def numpy_type(self): + return object + + def _get_type(self): + + return OrderedDict([ + ('name', 'interval'), + ('unit', 'DAY_TIME'), + ]) + + def generate_column(self, size, name=None): + min_day_value, max_day_value = -10000*366, 10000*366 + values = [{'days': random.randint(min_day_value, max_day_value), + 'milliseconds': random.randint(-86400000, +86400000)} + for _ in range(size)] + + is_valid = self._make_is_valid(size) + if name is None: + name = self.name + return PrimitiveColumn(name, size, is_valid, values) + + class FloatingPointType(PrimitiveType): def __init__(self, name, bit_width, nullable=True): @@ -875,13 +937,27 @@ def generate_datetime_case(): TimestampType('f11', 's', tz='UTC'), TimestampType('f12', 'ms', tz='US/Eastern'), TimestampType('f13', 'us', tz='Europe/Paris'), - TimestampType('f14', 'ns', tz='US/Pacific') + TimestampType('f14', 'ns', tz='US/Pacific'), ] batch_sizes = [7, 10] return _generate_file("datetime", fields, batch_sizes) +def generate_interval_case(): + fields = [ + DurationIntervalType('f1', 's'), + DurationIntervalType('f2', 'ms'), + DurationIntervalType('f3', 'us'), + DurationIntervalType('f4', 'ns'), + YearMonthIntervalType('f5'), + DayTimeIntervalType('f6'), + ] + + batch_sizes = [7, 10] + return _generate_file("interval", fields, batch_sizes) + + def generate_nested_case(): fields = [ ListType('list_nullable', get_field('item', 'int32')), @@ -926,6 +1002,7 @@ def _temp_path(): generate_primitive_case([0, 0, 0], name='primitive_zerolength'), generate_decimal_case(), generate_datetime_case(), + generate_interval_case(), generate_nested_case(), generate_dictionary_case().skip_category(SKIP_FLIGHT), ] @@ -993,6 +1070,12 @@ def _compare_implementations(self, producer, consumer): file_id = guid()[:8] + if ('JS' in (producer.name, consumer.name) and + "interval" in test_case.name): + print('TODO(ARROW-5239): Enable interval tests ' + + ' for JS once, JS supports them') + continue + # Make the random access file producer_file_path = os.path.join(self.temp_dir, file_id + '_' + name + '.json_as_file') diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd index 0b7eb918231..8c4702ae9e3 100644 --- a/java/vector/src/main/codegen/data/ArrowTypes.tdd +++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd @@ -94,6 +94,11 @@ name: "Interval", fields: [{name: "unit", type: short, valueType: IntervalUnit}], complex: false + }, + { + name: "Duration", + fields: [{name: "unit", type: short, valueType: TimeUnit}], + complex: false } ] } diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd index 329422f8ce2..cf8413cd47d 100644 --- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd +++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd @@ -76,6 +76,10 @@ { class: "UInt8" }, { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}] }, { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" }, + { class: "Duration", javaType: "long", friendlyType: "Duration", + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Duration", + typeParams: [ {name: "unit", type: "org.apache.arrow.vector.types.TimeUnit"} ], + arrowTypeConstructorParams: ["unit"]} { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl index 920c528cca1..de437b4b081 100644 --- a/java/vector/src/main/codegen/includes/vv_imports.ftl +++ b/java/vector/src/main/codegen/includes/vv_imports.ftl @@ -60,6 +60,3 @@ import java.time.Period; import java.time.ZonedDateTime; - - - diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java index 629bb73a76c..fcea3e882f0 100644 --- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java +++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java @@ -116,6 +116,8 @@ public void read(Nullable${name}Holder h) { return Duration.ofDays(holder.days).plusMillis(holder.milliseconds); <#elseif minor.class == "IntervalYear"> return Period.ofMonths(holder.value); + <#elseif minor.class == "Duration"> + return DurationVector.toDuration(holder.value, holder.unit); <#elseif minor.class == "Bit" > return new Boolean(holder.value != 0); <#elseif minor.class == "Decimal"> diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java new file mode 100644 index 00000000000..51ea1a13e8b --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/DurationVector.java @@ -0,0 +1,426 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; + +import java.time.Duration; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.complex.impl.DurationReaderImpl; +import org.apache.arrow.vector.complex.reader.FieldReader; +import org.apache.arrow.vector.holders.DurationHolder; +import org.apache.arrow.vector.holders.NullableDurationHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.Types.MinorType; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.util.TransferPair; + +import io.netty.buffer.ArrowBuf; + +/** + * DurationVector implements a fixed width vector (8 bytes) of + * a configurable TimeUnit granularity duration values which could be null. + * A validity buffer (bit vector) is maintained to track which elements in the + * vector are null. + */ +public class DurationVector extends BaseFixedWidthVector { + private static final byte TYPE_WIDTH = 8; + private final FieldReader reader; + private final TimeUnit unit; + + /** + * Instantiate a DurationVector. This doesn't allocate any memory for + * the data in vector. + * + * @param name name of the vector + * @param fieldType type of Field materialized by this vector + * @param allocator allocator for memory management. + */ + public DurationVector(String name, FieldType fieldType, BufferAllocator allocator) { + super(name, allocator, fieldType, TYPE_WIDTH); + reader = new DurationReaderImpl(DurationVector.this); + this.unit = ((ArrowType.Duration)fieldType.getType()).getUnit(); + + } + + /** + * Get a reader that supports reading values from this vector. + * + * @return Field Reader for this vector + */ + @Override + public FieldReader getReader() { + return reader; + } + + /** + * Get minor type for this vector. The vector holds values belonging + * to a particular type. + * + * @return {@link MinorType} + */ + @Override + public MinorType getMinorType() { + return MinorType.DURATION; + } + + + /*----------------------------------------------------------------* + | | + | vector value retrieval methods | + | | + *----------------------------------------------------------------*/ + + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + *

This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static long get(final ArrowBuf buffer, final int index) { + return buffer.getLong(index * TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector. + * + * @param index position of element + * @return element at given index + */ + public ArrowBuf get(int index) throws IllegalStateException { + if (isSet(index) == 0) { + return null; + } + return valueBuffer.slice(index * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Get the element at the given index from the vector and + * sets the state in holder. If element at given index + * is null, holder.isSet will be zero. + * + * @param index position of element + */ + public void get(int index, NullableDurationHolder holder) { + if (isSet(index) == 0) { + holder.isSet = 0; + return; + } + holder.isSet = 1; + holder.value = get(valueBuffer, index); + } + + /** + * Same as {@link #get(int)}. + * + * @param index position of element + * @return element at given index + */ + public Duration getObject(int index) { + if (isSet(index) == 0) { + return null; + } else { + final long value = get(valueBuffer, index); + return toDuration(value, unit); + } + } + + public static Duration toDuration(long value, TimeUnit unit) { + switch (unit) { + case SECOND: + return Duration.ofSeconds(value); + case MILLISECOND: + return Duration.ofMillis(value); + case NANOSECOND: + return Duration.ofNanos(value); + case MICROSECOND: + return Duration.ofNanos(MICROSECONDS.toNanos(value)); + default: + throw new IllegalArgumentException("Unknown timeunit: " + unit); + } + } + + /** + * Get the Interval value at a given index as a {@link StringBuilder} object. + * + * @param index position of the element + * @return String Builder object with Interval in java.time.Duration format. + */ + public StringBuilder getAsStringBuilder(int index) { + if (isSet(index) == 0) { + return null; + } else { + return getAsStringBuilderHelper(index); + } + } + + private StringBuilder getAsStringBuilderHelper(int index) { + return new StringBuilder(getObject(index).toString()); + } + + /** + * Copy a cell value from a particular index in source vector to a particular + * position in this vector. + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFrom(int fromIndex, int thisIndex, DurationVector from) { + BitVectorHelper.setValidityBit(validityBuffer, thisIndex, from.isSet(fromIndex)); + from.valueBuffer.getBytes(fromIndex * TYPE_WIDTH, this.valueBuffer, + thisIndex * TYPE_WIDTH, TYPE_WIDTH); + } + + /** + * Same as {@link #copyFrom(int, int, DurationVector)} except that + * it handles the case when the capacity of the vector needs to be expanded + * before copy. + * + * @param fromIndex position to copy from in source vector + * @param thisIndex position to copy to in this vector + * @param from source vector + */ + public void copyFromSafe(int fromIndex, int thisIndex, DurationVector from) { + handleSafe(thisIndex); + copyFrom(fromIndex, thisIndex, from); + } + + + /*----------------------------------------------------------------* + | | + | vector value setter methods | + | | + *----------------------------------------------------------------*/ + + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value value of element + */ + public void set(int index, ArrowBuf value) { + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setBytes(index * TYPE_WIDTH, value, 0, TYPE_WIDTH); + } + + /** + * Set the element at the given index to the given value. + * + * @param index position of element + * @param value The duration value (in the timeunit associated with this vector) + */ + public void set(int index, long value) { + final int offsetIndex = index * TYPE_WIDTH; + BitVectorHelper.setValidityBitToOne(validityBuffer, index); + valueBuffer.setLong(offsetIndex, value); + } + + /** + * Set the element at the given index to the value set in data holder. + * If the value in holder is not indicated as set, element in the + * at the given index will be null. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void set(int index, NullableDurationHolder holder) throws IllegalArgumentException { + if (holder.isSet < 0) { + throw new IllegalArgumentException(); + } else if (holder.isSet > 0) { + set(index, holder.value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Set the element at the given index to the value set in data holder. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void set(int index, DurationHolder holder) { + set(index, holder.value); + } + + /** + * Same as {@link #set(int, ArrowBuf)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value value of element + */ + public void setSafe(int index, ArrowBuf value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, long)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param value duration in the time unit this vector was constructed with + */ + public void setSafe(int index, long value) { + handleSafe(index); + set(index, value); + } + + /** + * Same as {@link #set(int, NullableDurationHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder nullable data holder for value of element + */ + public void setSafe(int index, NullableDurationHolder holder) throws IllegalArgumentException { + handleSafe(index); + set(index, holder); + } + + /** + * Same as {@link #set(int, DurationHolder)} except that it handles the + * case when index is greater than or equal to existing + * value capacity {@link #getValueCapacity()}. + * + * @param index position of element + * @param holder data holder for value of element + */ + public void setSafe(int index, DurationHolder holder) { + handleSafe(index); + set(index, holder); + } + + /** + * Set the element at the given index to null. + * + * @param index position of element + */ + public void setNull(int index) { + handleSafe(index); + // not really needed to set the bit to 0 as long as + // the buffer always starts from 0. + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + + /** + * Store the given value at a particular position in the vector. isSet indicates + * whether the value is NULL or not. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value The duration value (in the TimeUnit associated with this vector). + */ + public void set(int index, int isSet, long value) { + if (isSet > 0) { + set(index, value); + } else { + BitVectorHelper.setValidityBit(validityBuffer, index, 0); + } + } + + /** + * Same as {@link #set(int, int, long)} except that it handles the case + * when index is greater than or equal to current value capacity of the + * vector. + * + * @param index position of the new value + * @param isSet 0 for NULL value, 1 otherwise + * @param value The duration value (in the timeunit associated with this vector) + */ + public void setSafe(int index, int isSet, long value) { + handleSafe(index); + set(index, isSet, value); + } + + + /*----------------------------------------------------------------* + | | + | vector transfer | + | | + *----------------------------------------------------------------*/ + + + /** + * Construct a TransferPair comprising of this and and a target vector of + * the same type. + * + * @param ref name of the target vector + * @param allocator allocator for the target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { + return new TransferImpl(ref, allocator); + } + + /** + * Construct a TransferPair with a desired target vector of the same type. + * + * @param to target vector + * @return {@link TransferPair} + */ + @Override + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((DurationVector) to); + } + + private class TransferImpl implements TransferPair { + DurationVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new DurationVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(DurationVector to) { + this.to = to; + } + + @Override + public DurationVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, DurationVector.this); + } + } +} diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java index c74ac460f75..3afe757d199 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalDayVector.java @@ -39,7 +39,7 @@ * vector are null. */ public class IntervalDayVector extends BaseFixedWidthVector { - private static final byte TYPE_WIDTH = 8; + public static final byte TYPE_WIDTH = 8; private static final byte MILLISECOND_OFFSET = 4; private final FieldReader reader; @@ -95,6 +95,33 @@ public MinorType getMinorType() { | | *----------------------------------------------------------------*/ + /** + * Given a data buffer, get the number of days stored at a particular position + * in the vector. + * + *

This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return day value stored at the index. + */ + public static int getDays(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + + /** + * Given a data buffer, get the get the number of milliseconds stored at a particular position + * in the vector. + * + *

This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return milliseconds value stored at the index. + */ + public static int getMilliseconds(final ArrowBuf buffer, final int index) { + return buffer.getInt((index * TYPE_WIDTH) + MILLISECOND_OFFSET); + } /** * Get the element at the given index from the vector. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java index 13a3ca16fe8..ee19ba6140b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/IntervalYearVector.java @@ -30,6 +30,8 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; +import io.netty.buffer.ArrowBuf; + /** * IntervalYearVector implements a fixed width (4 bytes) vector of * interval (years and months) values which could be null. A validity buffer @@ -92,6 +94,20 @@ public MinorType getMinorType() { *----------------------------------------------------------------*/ + /** + * Given a data buffer, get the value stored at a particular position + * in the vector. + * + *

This method should not be used externally. + * + * @param buffer data buffer + * @param index position of the element. + * @return value stored at the index. + */ + public static int getTotalMonths(final ArrowBuf buffer, final int index) { + return buffer.getInt(index * TYPE_WIDTH); + } + /** * Get the element at the given index from the vector. * diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java index 1a639ce2438..b53a5bb571d 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java @@ -31,6 +31,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Bool; import org.apache.arrow.vector.types.pojo.ArrowType.Date; import org.apache.arrow.vector.types.pojo.ArrowType.Decimal; +import org.apache.arrow.vector.types.pojo.ArrowType.Duration; import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary; import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList; import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint; @@ -202,6 +203,11 @@ public TypeLayout visit(Interval type) { } } + @Override + public TypeLayout visit(Duration type) { + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(64)); + } + }); return layout; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java index e5fca8cc442..3e2c8da7e53 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java @@ -17,15 +17,9 @@ package org.apache.arrow.vector.ipc; -import static com.fasterxml.jackson.core.JsonToken.END_ARRAY; -import static com.fasterxml.jackson.core.JsonToken.END_OBJECT; -import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; -import static com.fasterxml.jackson.core.JsonToken.START_OBJECT; +import static com.fasterxml.jackson.core.JsonToken.*; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.arrow.vector.BufferLayout.BufferType.DATA; -import static org.apache.arrow.vector.BufferLayout.BufferType.OFFSET; -import static org.apache.arrow.vector.BufferLayout.BufferType.TYPE; -import static org.apache.arrow.vector.BufferLayout.BufferType.VALIDITY; +import static org.apache.arrow.vector.BufferLayout.BufferType.*; import java.io.File; import java.io.IOException; @@ -47,6 +41,7 @@ import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.IntervalDayVector; import org.apache.arrow.vector.SmallIntVector; import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.TypeLayout; @@ -234,6 +229,23 @@ protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException } }; + BufferReader DAY_MILLIS = new BufferReader() { + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + final int size = count * IntervalDayVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); + + for (int i = 0; i < count; i++) { + readToken(START_OBJECT); + buf.writeInt(readNextField("days", Integer.class)); + buf.writeInt(readNextField("milliseconds", Integer.class)); + readToken(END_OBJECT); + } + + return buf; + } + }; + BufferReader INT1 = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { @@ -493,6 +505,15 @@ private ArrowBuf readIntoBuffer(BufferAllocator allocator, BufferType bufferType case TIMESTAMPSECTZ: reader = helper.INT8; break; + case INTERVALYEAR: + reader = helper.INT4; + break; + case INTERVALDAY: + reader = helper.DAY_MILLIS; + break; + case DURATION: + reader = helper.INT8; + break; default: throw new UnsupportedOperationException("Cannot read array of type " + type); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index acf452357dd..46afe11c7d7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -17,10 +17,7 @@ package org.apache.arrow.vector.ipc; -import static org.apache.arrow.vector.BufferLayout.BufferType.DATA; -import static org.apache.arrow.vector.BufferLayout.BufferType.OFFSET; -import static org.apache.arrow.vector.BufferLayout.BufferType.TYPE; -import static org.apache.arrow.vector.BufferLayout.BufferType.VALIDITY; +import static org.apache.arrow.vector.BufferLayout.BufferType.*; import java.io.File; import java.io.IOException; @@ -38,11 +35,14 @@ import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DateMilliVector; import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.DurationVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.IntervalDayVector; +import org.apache.arrow.vector.IntervalYearVector; import org.apache.arrow.vector.SmallIntVector; import org.apache.arrow.vector.TimeMicroVector; import org.apache.arrow.vector.TimeMilliVector; @@ -302,6 +302,18 @@ private void writeValueToGenerator( case TIMESTAMPNANOTZ: generator.writeNumber(TimeStampNanoTZVector.get(buffer, index)); break; + case DURATION: + generator.writeNumber(DurationVector.get(buffer, index)); + break; + case INTERVALYEAR: + generator.writeNumber(IntervalYearVector.getTotalMonths(buffer, index)); + break; + case INTERVALDAY: + generator.writeStartObject(); + generator.writeObjectField("days", IntervalDayVector.getDays(buffer, index)); + generator.writeObjectField("milliseconds", IntervalDayVector.getMilliseconds(buffer, index)); + generator.writeEndObject(); + break; case BIT: generator.writeNumber(BitVectorHelper.get(buffer, index)); break; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index a505821f0bc..bb94a80973a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -27,6 +27,7 @@ import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DateMilliVector; import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.DurationVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.arrow.vector.Float4Vector; @@ -65,6 +66,7 @@ import org.apache.arrow.vector.complex.impl.DateDayWriterImpl; import org.apache.arrow.vector.complex.impl.DateMilliWriterImpl; import org.apache.arrow.vector.complex.impl.DecimalWriterImpl; +import org.apache.arrow.vector.complex.impl.DurationWriterImpl; import org.apache.arrow.vector.complex.impl.FixedSizeBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.Float4WriterImpl; import org.apache.arrow.vector.complex.impl.Float8WriterImpl; @@ -101,6 +103,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Bool; import org.apache.arrow.vector.types.pojo.ArrowType.Date; import org.apache.arrow.vector.types.pojo.ArrowType.Decimal; +import org.apache.arrow.vector.types.pojo.ArrowType.Duration; import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary; import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList; import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint; @@ -378,6 +381,23 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { return new IntervalDayWriterImpl((IntervalDayVector) vector); } }, + DURATION(null) { + @Override + public FieldVector getNewVector( + String name, + FieldType fieldType, + BufferAllocator allocator, + CallBack schemaChangeCallback) { + return new DurationVector(name, fieldType, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new DurationWriterImpl((DurationVector) vector); + } + }, + + INTERVALYEAR(new Interval(IntervalUnit.YEAR_MONTH)) { @Override public FieldVector getNewVector( @@ -831,6 +851,11 @@ public MinorType visit(Interval type) { throw new IllegalArgumentException("unknown unit: " + type); } } + + @Override + public MinorType visit(Duration type) { + return MinorType.DURATION; + } }); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDurationVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDurationVector.java new file mode 100644 index 00000000000..8ae876f2011 --- /dev/null +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDurationVector.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.vector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.time.Duration; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.holders.NullableDurationHolder; +import org.apache.arrow.vector.types.TimeUnit; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestDurationVector { + RootAllocator allocator; + + @Before + public void init() { + allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); + } + + @After + public void terminate() { + allocator.close(); + } + + @Test + public void testSecBasics() { + try (DurationVector secVector = TestUtils.newVector(DurationVector.class, "second", + new ArrowType.Duration(TimeUnit.SECOND), allocator)) { + + secVector.allocateNew(); + secVector.setNull(0); + secVector.setSafe(1, 1000); + secVector.setValueCount(2); + assertNull(secVector.getObject(0)); + assertEquals(Duration.ofSeconds(1000), secVector.getObject(1)); + assertNull(secVector.getAsStringBuilder(0)); + assertEquals("PT16M40S", secVector.getAsStringBuilder(1).toString()); + // Holder + NullableDurationHolder holder = new NullableDurationHolder(); + secVector.get(0, holder); + assertEquals(0, holder.isSet); + secVector.get(1, holder); + assertEquals(1 , holder.isSet); + assertEquals(1000 , holder.value); + } + } + + @Test + public void testMilliBasics() { + try (DurationVector milliVector = TestUtils.newVector(DurationVector.class, "nanos", + new ArrowType.Duration(TimeUnit.MILLISECOND), allocator)) { + + milliVector.allocateNew(); + milliVector.setNull(0); + milliVector.setSafe(1, 1000); + milliVector.setValueCount(2); + assertNull(milliVector.getObject(0)); + assertEquals(Duration.ofSeconds(1), milliVector.getObject(1)); + assertNull(milliVector.getAsStringBuilder(0)); + assertEquals("PT1S", milliVector.getAsStringBuilder(1).toString()); + // Holder + NullableDurationHolder holder = new NullableDurationHolder(); + milliVector.get(0, holder); + assertEquals(0, holder.isSet); + milliVector.get(1, holder); + assertEquals(1 , holder.isSet); + assertEquals(1000 , holder.value); + } + } + + @Test + public void testMicroBasics() { + try (DurationVector microVector = TestUtils.newVector(DurationVector.class, "micro", + new ArrowType.Duration(TimeUnit.MICROSECOND), allocator)) { + + microVector.allocateNew(); + microVector.setNull(0); + microVector.setSafe(1, 1000); + microVector.setValueCount(2); + assertNull(microVector.getObject(0)); + assertEquals(Duration.ofMillis(1), microVector.getObject(1)); + assertNull(microVector.getAsStringBuilder(0)); + assertEquals("PT0.001S", microVector.getAsStringBuilder(1).toString()); + // Holder + NullableDurationHolder holder = new NullableDurationHolder(); + microVector.get(0, holder); + assertEquals(0, holder.isSet); + microVector.get(1, holder); + assertEquals(1 , holder.isSet); + assertEquals(1000 , holder.value); + } + } + + @Test + public void testNanosBasics() { + try (DurationVector nanoVector = TestUtils.newVector(DurationVector.class, "nanos", + new ArrowType.Duration(TimeUnit.NANOSECOND), allocator)) { + + nanoVector.allocateNew(); + nanoVector.setNull(0); + nanoVector.setSafe(1, 1000000); + nanoVector.setValueCount(2); + assertNull(nanoVector.getObject(0)); + assertEquals(Duration.ofMillis(1), nanoVector.getObject(1)); + assertNull(nanoVector.getAsStringBuilder(0)); + assertEquals("PT0.001S", nanoVector.getAsStringBuilder(1).toString()); + // Holder + NullableDurationHolder holder = new NullableDurationHolder(); + nanoVector.get(0, holder); + assertEquals(0, holder.isSet); + nanoVector.get(1, holder); + assertEquals(1 , holder.isSet); + assertEquals(1000000 , holder.value); + } + } +} diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java index 2b109dce77a..3ca8d0af6a6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java @@ -32,6 +32,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Bool; import org.apache.arrow.vector.types.pojo.ArrowType.Date; import org.apache.arrow.vector.types.pojo.ArrowType.Decimal; +import org.apache.arrow.vector.types.pojo.ArrowType.Duration; import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeBinary; import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint; import org.apache.arrow.vector.types.pojo.ArrowType.Int; @@ -66,13 +67,14 @@ public void testComplex() throws IOException { field("f", new FloatingPoint(FloatingPointPrecision.SINGLE)), field("g", new Timestamp(TimeUnit.MILLISECOND, "UTC")), field("h", new Timestamp(TimeUnit.MICROSECOND, null)), - field("i", new Interval(IntervalUnit.DAY_TIME)) + field("i", new Interval(IntervalUnit.DAY_TIME)), + field("j", new ArrowType.Duration(TimeUnit.SECOND)) )); roundTrip(schema); assertEquals( "Schema, e: List, " + "f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND, UTC), h: Timestamp(MICROSECOND, null), " + - "i: Interval(DAY_TIME)>", + "i: Interval(DAY_TIME), j: Duration(SECOND)>", schema.toString()); } @@ -98,7 +100,9 @@ public void testAll() throws IOException { field("q", new Timestamp(TimeUnit.MILLISECOND, "UTC")), field("r", new Timestamp(TimeUnit.MICROSECOND, null)), field("s", new Interval(IntervalUnit.DAY_TIME)), - field("t", new FixedSizeBinary(100)) + field("t", new FixedSizeBinary(100)), + field("u", new Duration(TimeUnit.SECOND)), + field("v", new Duration(TimeUnit.MICROSECOND)) )); roundTrip(schema); } @@ -168,6 +172,18 @@ public void testInterval() throws IOException { contains(schema, "YEAR_MONTH", "DAY_TIME"); } + @Test + public void testRoundTripDurationInterval() throws IOException { + Schema schema = new Schema(asList( + field("a", new Duration(TimeUnit.SECOND)), + field("b", new Duration(TimeUnit.MILLISECOND)), + field("c", new Duration(TimeUnit.MICROSECOND)), + field("d", new Duration(TimeUnit.NANOSECOND)) + )); + roundTrip(schema); + contains(schema, "SECOND", "MILLI", "MICRO", "NANO"); + } + @Test public void testFP() throws IOException { Schema schema = new Schema(asList( @@ -207,7 +223,7 @@ private void validateHashCode(Object o1, Object o2) { assertEquals(o1 + " == " + o2, o1.hashCode(), o2.hashCode()); } - private void contains(Schema schema, String... s) throws IOException { + private void contains(Schema schema, String... s) { String json = schema.toJson(); for (String string : s) { assertTrue(json + " contains " + string, json.contains(string)); From daf8b4fb6d64fb0847a4159f96c030e951ef7258 Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Wed, 15 May 2019 13:33:28 -0700 Subject: [PATCH 2/3] duration_type->duration --- cpp/src/arrow/ipc/json-internal.cc | 2 +- cpp/src/arrow/ipc/metadata-internal.cc | 2 +- cpp/src/arrow/ipc/test-common.cc | 6 +++--- cpp/src/arrow/type-test.cc | 8 ++++---- cpp/src/arrow/type.cc | 2 +- cpp/src/arrow/type.h | 2 +- cpp/src/arrow/vendored/datetime/date.h | 10 +++++----- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc index bd64b914a47..a26eac016fb 100644 --- a/cpp/src/arrow/ipc/json-internal.cc +++ b/cpp/src/arrow/ipc/json-internal.cc @@ -819,7 +819,7 @@ static Status GetDuration(const RjObject& json_type, std::shared_ptr* TimeUnit::type unit; RETURN_NOT_OK(GetUnitFromString(unit_str, &unit)); - *type = duration_type(unit); + *type = duration(unit); return Status::OK(); } diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc index 46b99b12554..6195ca54c36 100644 --- a/cpp/src/arrow/ipc/metadata-internal.cc +++ b/cpp/src/arrow/ipc/metadata-internal.cc @@ -288,7 +288,7 @@ static Status ConcreteTypeFromFlatbuffer( case flatbuf::Type_Duration: { auto duration = static_cast(type_data); TimeUnit::type unit = FromFlatbufferUnit(duration->unit()); - *out = duration_type(unit); + *out = arrow::duration(unit); return Status::OK(); } diff --git a/cpp/src/arrow/ipc/test-common.cc b/cpp/src/arrow/ipc/test-common.cc index 555d7ca7275..ff7ce051cef 100644 --- a/cpp/src/arrow/ipc/test-common.cc +++ b/cpp/src/arrow/ipc/test-common.cc @@ -582,9 +582,9 @@ Status MakeTimestamps(std::shared_ptr* out) { Status MakeIntervals(std::shared_ptr* out) { std::vector is_valid = {true, true, true, false, true, true, true}; - auto f0 = field("f0", duration_type(TimeUnit::MILLI)); - auto f1 = field("f1", duration_type(TimeUnit::NANO)); - auto f2 = field("f2", duration_type(TimeUnit::SECOND)); + auto f0 = field("f0", duration(TimeUnit::MILLI)); + auto f1 = field("f1", duration(TimeUnit::NANO)); + auto f2 = field("f2", duration(TimeUnit::SECOND)); auto f3 = field("f3", day_time_interval()); auto f4 = field("f4", month_interval()); auto schema = ::arrow::schema({f0, f1, f2, f3, f4}); diff --git a/cpp/src/arrow/type-test.cc b/cpp/src/arrow/type-test.cc index ad6667a0b57..aeffb8f42ff 100644 --- a/cpp/src/arrow/type-test.cc +++ b/cpp/src/arrow/type-test.cc @@ -473,10 +473,10 @@ TEST(TestDurationType, Equals) { } TEST(TestDurationType, ToString) { - auto t1 = duration_type(TimeUnit::MILLI); - auto t2 = duration_type(TimeUnit::NANO); - auto t3 = duration_type(TimeUnit::SECOND); - auto t4 = duration_type(TimeUnit::MICRO); + auto t1 = duration(TimeUnit::MILLI); + auto t2 = duration(TimeUnit::NANO); + auto t3 = duration(TimeUnit::SECOND); + auto t4 = duration(TimeUnit::MICRO); ASSERT_EQ("duration[ms]", t1->ToString()); ASSERT_EQ("duration[ns]", t2->ToString()); diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc index 9ef6a9bddd9..58b8cb3d150 100644 --- a/cpp/src/arrow/type.cc +++ b/cpp/src/arrow/type.cc @@ -554,7 +554,7 @@ std::shared_ptr fixed_size_binary(int32_t byte_width) { return std::make_shared(byte_width); } -std::shared_ptr duration_type(TimeUnit::type unit) { +std::shared_ptr duration(TimeUnit::type unit) { return std::make_shared(unit); } diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index fcd86378d30..dff2bbe8469 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -993,7 +993,7 @@ std::shared_ptr fixed_size_list(const std::shared_ptr& value int32_t list_size); /// \brief Return an Duration instance (naming use _type to avoid namespace conflict with /// built in time clases). -std::shared_ptr ARROW_EXPORT duration_type(TimeUnit::type unit); +std::shared_ptr ARROW_EXPORT duration(TimeUnit::type unit); /// \brief Return an DayTimeIntervalType instance std::shared_ptr ARROW_EXPORT day_time_interval(); diff --git a/cpp/src/arrow/vendored/datetime/date.h b/cpp/src/arrow/vendored/datetime/date.h index f2889e416b0..4581d1a459f 100644 --- a/cpp/src/arrow/vendored/datetime/date.h +++ b/cpp/src/arrow/vendored/datetime/date.h @@ -1109,7 +1109,7 @@ trunc(const std::chrono::duration& d) { using namespace std::chrono; using rep = typename std::common_type::type; - return To{detail::trunc(duration_cast(duration_cast>(d)).count())}; + return To{detail::trunc(duration_cast(duration_cast>(d)).count())}; } #ifndef HAS_CHRONO_ROUNDING @@ -1155,7 +1155,7 @@ floor(const std::chrono::duration& d) { using namespace std::chrono; using rep = typename std::common_type::type; - return floor(floor>(d)); + return floor(floor>(d)); } // round to nearest, to even on tie @@ -6127,7 +6127,7 @@ from_stream(std::basic_istream& is, const CharT* fmt, { h = hours{H}; min = minutes{M}; - s = round(duration{S}); + s = round(std::chrono::system_clock::duration{S}); } #endif command = nullptr; @@ -6508,7 +6508,7 @@ from_stream(std::basic_istream& is, const CharT* fmt, long double S; read(is, rld{S, 1, width == -1 ? w : static_cast(width)}); if (!is.fail()) - s = round(duration{S}); + s = round(std::chrono::duration{S}); #if !ONLY_C_LOCALE } else if (modified == CharT{'O'}) @@ -6545,7 +6545,7 @@ from_stream(std::basic_istream& is, const CharT* fmt, { h = hours{H}; min = minutes{M}; - s = round(duration{S}); + s = round(std::chrono::duration{S}); } } else From cc6d39c89c8ce4643f0674e940d0d4cbc2d65ec7 Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Wed, 15 May 2019 13:36:32 -0700 Subject: [PATCH 3/3] remove system clock --- cpp/src/arrow/vendored/datetime/date.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/vendored/datetime/date.h b/cpp/src/arrow/vendored/datetime/date.h index 4581d1a459f..3aef9f7443a 100644 --- a/cpp/src/arrow/vendored/datetime/date.h +++ b/cpp/src/arrow/vendored/datetime/date.h @@ -6127,7 +6127,7 @@ from_stream(std::basic_istream& is, const CharT* fmt, { h = hours{H}; min = minutes{M}; - s = round(std::chrono::system_clock::duration{S}); + s = round(std::chrono::duration{S}); } #endif command = nullptr;