diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh index 56f5ab7d2d3..6e93ed79a22 100755 --- a/ci/travis_script_integration.sh +++ b/ci/travis_script_integration.sh @@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh pushd $ARROW_JAVA_DIR -mvn package +echo "mvn package" +mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false) popd diff --git a/integration/integration_test.py b/integration/integration_test.py index 661f5c97770..646646997f7 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -544,7 +544,8 @@ def get_json(self): class JSONFile(object): - def __init__(self, schema, batches): + def __init__(self, name, schema, batches): + self.name = name self.schema = schema self.batches = batches @@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True): raise TypeError(dtype) -def _generate_file(fields, batch_sizes): +def _generate_file(name, fields, batch_sizes): schema = JSONSchema(fields) batches = [] for size in batch_sizes: @@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes): batches.append(JSONRecordBatch(size, columns)) - return JSONFile(schema, batches) + return JSONFile(name, schema, batches) def generate_primitive_case(batch_sizes): @@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes): fields.append(get_field(type_ + "_nullable", type_, True)) fields.append(get_field(type_ + "_nonnullable", type_, False)) - return _generate_file(fields, batch_sizes) + return _generate_file("primitive", fields, batch_sizes) def generate_datetime_case(): @@ -619,11 +620,11 @@ def generate_datetime_case(): TimestampType('f7', 'ms'), TimestampType('f8', 'us'), TimestampType('f9', 'ns'), - TimestampType('f10', 'ms', tz='America/New_York') + TimestampType('f10', 'ms', tz=None) ] batch_sizes = [7, 10] - return _generate_file(fields, batch_sizes) + return _generate_file("datetime", fields, batch_sizes) def generate_nested_case(): @@ -637,7 +638,7 @@ def generate_nested_case(): ] batch_sizes = [7, 10] - return _generate_file(fields, batch_sizes) + return _generate_file("nested", fields, batch_sizes) def get_generated_json_files(): @@ -655,7 +656,7 @@ def _temp_path(): generated_paths = [] for file_obj in file_objs: - out_path = os.path.join(temp_dir, guid() + '.json') + out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json') file_obj.write(out_path) generated_paths.append(out_path) @@ -684,11 +685,15 @@ def _compare_implementations(self, producer, consumer): consumer.name)) for json_path in self.json_files: + print('=====================================================================================') print('Testing file {0}'.format(json_path)) + print('=====================================================================================') + + name = os.path.splitext(os.path.basename(json_path))[0] # Make the random access file print('-- Creating binary inputs') - producer_file_path = os.path.join(self.temp_dir, guid()) + producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow') producer.json_to_file(json_path, producer_file_path) # Validate the file @@ -696,8 +701,8 @@ def _compare_implementations(self, producer, consumer): consumer.validate(json_path, producer_file_path) print('-- Validating stream') - producer_stream_path = os.path.join(self.temp_dir, guid()) - consumer_file_path = os.path.join(self.temp_dir, guid()) + producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream') + consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow') producer.file_to_stream(producer_file_path, producer_stream_path) consumer.stream_to_file(producer_stream_path, diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd index d472b559347..ca6d9ecbe85 100644 --- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd +++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd @@ -59,7 +59,7 @@ { class: "DateDay" }, { class: "IntervalYear", javaType: "int", friendlyType: "Period" }, { class: "TimeSec" }, - { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" } + { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" } ] }, { @@ -71,12 +71,12 @@ minor: [ { class: "BigInt"}, { class: "UInt8" }, - { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], }, - { class: "DateMilli", javaType: "long", friendlyType: "DateTime" }, - { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }, - { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }, - { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }, - { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }, + { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], }, + { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" }, + { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, { class: "TimeMicro" }, { class: "TimeNano" } ] diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl index 9b4b79bfd7b..e723e7d7ea3 100644 --- a/java/vector/src/main/codegen/includes/vv_imports.ftl +++ b/java/vector/src/main/codegen/includes/vv_imports.ftl @@ -57,6 +57,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import org.joda.time.DateTime; +import org.joda.time.LocalDateTime; import org.joda.time.Period; diff --git a/java/vector/src/main/codegen/templates/AbstractFieldReader.java b/java/vector/src/main/codegen/templates/AbstractFieldReader.java index 79d4c122f0e..b16ee162fde 100644 --- a/java/vector/src/main/codegen/templates/AbstractFieldReader.java +++ b/java/vector/src/main/codegen/templates/AbstractFieldReader.java @@ -26,8 +26,8 @@ <#include "/@includes/vv_imports.ftl" /> -/* - * This class is generated using freemarker and the ${.template_name} template. +/** + * Source code generated using FreeMarker template ${.template_name} */ @SuppressWarnings("unused") abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{ @@ -51,7 +51,7 @@ public Field getField() { } <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", - "Character", "DateTime", "Period", "Double", "Float", + "Character", "LocalDateTime", "Period", "Double", "Float", "Text", "String", "Byte", "Short", "byte[]"] as friendlyType> <#assign safeType=friendlyType /> <#if safeType=="byte[]"><#assign safeType="ByteArray" /> diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java index dc99aad0bb3..93746303d93 100644 --- a/java/vector/src/main/codegen/templates/ArrowType.java +++ b/java/vector/src/main/codegen/templates/ArrowType.java @@ -38,6 +38,7 @@ /** * Arrow types + * Source code generated using FreeMarker template ${.template_name} **/ @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, diff --git a/java/vector/src/main/codegen/templates/BaseReader.java b/java/vector/src/main/codegen/templates/BaseReader.java index 72fea58d0bc..ea3286e8681 100644 --- a/java/vector/src/main/codegen/templates/BaseReader.java +++ b/java/vector/src/main/codegen/templates/BaseReader.java @@ -26,8 +26,9 @@ <#include "/@includes/vv_imports.ftl" /> - - +/** + * Source code generated using FreeMarker template ${.template_name} + */ @SuppressWarnings("unused") public interface BaseReader extends Positionable{ Field getField(); diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java index d53744539aa..38cd1bfdeb3 100644 --- a/java/vector/src/main/codegen/templates/ComplexReaders.java +++ b/java/vector/src/main/codegen/templates/ComplexReaders.java @@ -47,6 +47,9 @@ <#include "/@includes/vv_imports.ftl" /> +/** + * Source code generated using FreeMarker template ${.template_name} + */ @SuppressWarnings("unused") public class ${name}ReaderImpl extends AbstractFieldReader { @@ -123,12 +126,16 @@ public Object readObject(){ package org.apache.arrow.vector.complex.reader; <#include "/@includes/vv_imports.ftl" /> +/** + * Source code generated using FreeMarker template ${.template_name} + */ @SuppressWarnings("unused") public interface ${name}Reader extends BaseReader{ public void read(${minor.class?cap_first}Holder h); public void read(Nullable${minor.class?cap_first}Holder h); public Object readObject(); + // read friendly type public ${friendlyType} read${safeType}(); public boolean isSet(); public void copyAsValue(${minor.class}Writer writer); diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index 3457545cea5..c23b89d1bbc 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -139,6 +139,9 @@ public void writeNull() { package org.apache.arrow.vector.complex.writer; <#include "/@includes/vv_imports.ftl" /> +/* + * This class is generated using FreeMarker on the ${.template_name} template. + */ @SuppressWarnings("unused") public interface ${eName}Writer extends BaseWriter { public void write(${minor.class}Holder h); diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java index 05faaae1e9e..f403ecfac1f 100644 --- a/java/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java @@ -495,16 +495,14 @@ public long getTwoAsLong(int index) { <#elseif minor.class == "DateMilli"> @Override public ${friendlyType} getObject(int index) { - org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC); return date; } <#elseif minor.class == "TimeMilli"> @Override public ${friendlyType} getObject(int index) { - org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); - time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC); return time; } @@ -512,16 +510,14 @@ public long getTwoAsLong(int index) { @Override public ${friendlyType} getObject(int index) { long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index)); - org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC); return date; } <#elseif minor.class == "TimeStampMilli"> @Override public ${friendlyType} getObject(int index) { - org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC); return date; } @@ -530,8 +526,7 @@ public long getTwoAsLong(int index) { public ${friendlyType} getObject(int index) { // value is truncated when converting microseconds to milliseconds in order to use DateTime type long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index)); - org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC); return date; } @@ -540,8 +535,7 @@ public long getTwoAsLong(int index) { public ${friendlyType} getObject(int index) { // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index)); - org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC); - date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); + org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC); return date; } diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java index d66577bc1e4..e990fcc9334 100644 --- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java +++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java @@ -84,7 +84,7 @@ public boolean isSet() { } -@Override + @Override public void read(${name}Holder h) { <#list fields as field> h.${field.name} = holder.${field.name}; @@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) { h.isSet = isSet() ? 1 : 0; } - + // read friendly type @Override public ${friendlyType} read${safeType}(){ <#if nullMode == "Nullable"> @@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) { byte[] value = new byte [length]; holder.buffer.getBytes(holder.start, value, 0, length); -<#if minor.class == "VarBinary"> + <#if minor.class == "VarBinary"> return value; -<#elseif minor.class == "Var16Char"> + <#elseif minor.class == "Var16Char"> return new String(value); -<#elseif minor.class == "VarChar"> + <#elseif minor.class == "VarChar"> Text text = new Text(); text.set(value); return text; - + <#elseif minor.class == "Interval"> Period p = new Period(); diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java index ba0c088add7..7c75b3ae1df 100644 --- a/java/vector/src/main/codegen/templates/NullReader.java +++ b/java/vector/src/main/codegen/templates/NullReader.java @@ -29,7 +29,9 @@ <#include "/@includes/vv_imports.ftl" /> - +/** + * Source code generated using FreeMarker template ${.template_name} + */ @SuppressWarnings("unused") public class NullReader extends AbstractBaseReader implements FieldReader{ @@ -127,7 +129,7 @@ private void fail(String name){ } <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", - "Character", "DateTime", "Period", "Double", "Float", + "Character", "LocalDateTime", "Period", "Double", "Float", "Text", "String", "Byte", "Short", "byte[]"] as friendlyType> <#assign safeType=friendlyType /> <#if safeType=="byte[]"><#assign safeType="ByteArray" /> diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java index c56e95c89dc..0b5a209d40a 100644 --- a/java/vector/src/main/codegen/templates/UnionReader.java +++ b/java/vector/src/main/codegen/templates/UnionReader.java @@ -28,7 +28,9 @@ package org.apache.arrow.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> - +/** + * Source code generated using FreeMarker template ${.template_name} + */ @SuppressWarnings("unused") public class UnionReader extends AbstractFieldReader { @@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) { } <#list ["Object", "Integer", "Long", "Boolean", - "Character", "DateTime", "Double", "Float", + "Character", "LocalDateTime", "Double", "Float", "Text", "Byte", "Short", "byte[]"] as friendlyType> <#assign safeType=friendlyType /> <#if safeType=="byte[]"><#assign safeType="ByteArray" /> diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index d70cbae02bf..9d5dee5d237 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -50,6 +50,7 @@ * * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup * each time the vector is accessed. + * Source code generated using FreeMarker template ${.template_name} */ public class UnionVector implements FieldVector { diff --git a/java/vector/src/main/codegen/templates/ValueHolders.java b/java/vector/src/main/codegen/templates/ValueHolders.java index d744c523265..a474b691080 100644 --- a/java/vector/src/main/codegen/templates/ValueHolders.java +++ b/java/vector/src/main/codegen/templates/ValueHolders.java @@ -29,6 +29,9 @@ <#include "/@includes/vv_imports.ftl" /> +/** + * Source code generated using FreeMarker template ${.template_name} + */ public final class ${className} implements ValueHolder{ <#if mode.name == "Repeated"> diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 6023f1c9500..d5076d82c2a 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -241,7 +241,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { } }, // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC. - TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, "UTC")) { + TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new NullableTimeStampSecVector(name, fieldType, allocator); @@ -253,7 +253,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { } }, // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC. - TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, "UTC")) { + TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new NullableTimeStampMilliVector(name, fieldType, allocator); @@ -265,7 +265,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { } }, // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC. - TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, "UTC")) { + TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new NullableTimeStampMicroVector(name, fieldType, allocator); @@ -277,7 +277,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { } }, // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC. - TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, "UTC")) { + TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, null)) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new NullableTimeStampNanoVector(name, fieldType, allocator); @@ -580,6 +580,9 @@ public MinorType visit(FloatingPoint type) { } @Override public MinorType visit(Timestamp type) { + if (type.getTimezone() != null) { + throw new IllegalArgumentException("only timezone-less timestamps are supported for now: " + type); + } switch (type.getUnit()) { case SECOND: return MinorType.TIMESTAMPSEC; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java index 1f8ce069cf9..8aad41744f6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DateUtility.java @@ -18,6 +18,9 @@ package org.apache.arrow.vector.util; +import org.joda.time.DateTimeZone; +import org.joda.time.LocalDateTime; +import org.joda.time.LocalDateTimes; import org.joda.time.Period; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; @@ -679,4 +682,11 @@ public static int millisFromPeriod(final Period period){ (period.getMillis()); } + public static long toMillis(LocalDateTime localDateTime) { + return LocalDateTimes.getLocalMillis(localDateTime); + } + + public static int toMillisOfDay(final LocalDateTime localDateTime) { + return localDateTime.toDateTime(DateTimeZone.UTC).millisOfDay().get(); + } } diff --git a/java/vector/src/main/java/org/joda/time/LocalDateTimes.java b/java/vector/src/main/java/org/joda/time/LocalDateTimes.java new file mode 100644 index 00000000000..e4f999e1d82 --- /dev/null +++ b/java/vector/src/main/java/org/joda/time/LocalDateTimes.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.joda.time; + +/** + * Workaround to access package protected fields in JODA + * + */ +public class LocalDateTimes { + + public static long getLocalMillis(LocalDateTime localDateTime) { + return localDateTime.getLocalMillis(); + } + +} diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 99ba19bec80..aba65dbf374 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -56,8 +56,7 @@ import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; +import org.joda.time.LocalDateTime; import org.junit.Assert; import org.junit.Test; @@ -602,10 +601,10 @@ public void timeStampWriters() throws Exception { final long expectedMicros = 981173106123456L; final long expectedMillis = 981173106123L; final long expectedSecs = 981173106L; - final DateTime expectedSecDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 0).withZoneRetainFields(DateTimeZone.getDefault()); - final DateTime expectedMilliDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 123).withZoneRetainFields(DateTimeZone.getDefault()); - final DateTime expectedMicroDateTime = expectedMilliDateTime; - final DateTime expectedNanoDateTime = expectedMilliDateTime; + final LocalDateTime expectedSecDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 0); + final LocalDateTime expectedMilliDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123); + final LocalDateTime expectedMicroDateTime = expectedMilliDateTime; + final LocalDateTime expectedNanoDateTime = expectedMilliDateTime; // write MapVector parent = new MapVector("parent", allocator, null); @@ -650,28 +649,28 @@ public void timeStampWriters() throws Exception { FieldReader secReader = rootReader.reader("sec"); secReader.setPosition(0); - DateTime secDateTime = secReader.readDateTime(); + LocalDateTime secDateTime = secReader.readLocalDateTime(); Assert.assertEquals(expectedSecDateTime, secDateTime); long secLong = secReader.readLong(); Assert.assertEquals(expectedSecs, secLong); FieldReader milliReader = rootReader.reader("milli"); milliReader.setPosition(1); - DateTime milliDateTime = milliReader.readDateTime(); + LocalDateTime milliDateTime = milliReader.readLocalDateTime(); Assert.assertEquals(expectedMilliDateTime, milliDateTime); long milliLong = milliReader.readLong(); Assert.assertEquals(expectedMillis, milliLong); FieldReader microReader = rootReader.reader("micro"); microReader.setPosition(2); - DateTime microDateTime = microReader.readDateTime(); + LocalDateTime microDateTime = microReader.readLocalDateTime(); Assert.assertEquals(expectedMicroDateTime, microDateTime); long microLong = microReader.readLong(); Assert.assertEquals(expectedMicros, microLong); FieldReader nanoReader = rootReader.reader("nano"); nanoReader.setPosition(3); - DateTime nanoDateTime = nanoReader.readDateTime(); + LocalDateTime nanoDateTime = nanoReader.readLocalDateTime(); Assert.assertEquals(expectedNanoDateTime, nanoDateTime); long nanoLong = nanoReader.readLong(); Assert.assertEquals(expectedNanos, nanoLong); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java index 5ca083aa2df..5cc36a3b820 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java @@ -39,8 +39,8 @@ import org.apache.arrow.vector.complex.writer.TimeMilliWriter; import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter; import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder; -import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.joda.time.LocalDateTime; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -144,8 +144,8 @@ protected void validateComplexContent(int count, VectorSchemaRoot root) { } } - private DateTime makeDateTimeFromCount(int i) { - return new DateTime(2000 + i, 1 + i, 1 + i, i, i, i, i, DateTimeZone.UTC); + private LocalDateTime makeDateTimeFromCount(int i) { + return new LocalDateTime(2000 + i, 1 + i, 1 + i, i, i, i, i); } protected void writeDateTimeData(int count, NullableMapVector parent) { @@ -156,17 +156,17 @@ protected void writeDateTimeData(int count, NullableMapVector parent) { TimeMilliWriter timeWriter = rootWriter.timeMilli("time"); TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli"); for (int i = 0; i < count; i++) { - DateTime dt = makeDateTimeFromCount(i); + LocalDateTime dt = makeDateTimeFromCount(i); // Number of days in milliseconds since epoch, stored as 64-bit integer, only date part is used dateWriter.setPosition(i); - long dateLong = dt.minusMillis(dt.getMillisOfDay()).getMillis(); + long dateLong = org.apache.arrow.vector.util.DateUtility.toMillis(dt.minusMillis(dt.getMillisOfDay())); dateWriter.writeDateMilli(dateLong); // Time is a value in milliseconds since midnight, stored as 32-bit integer timeWriter.setPosition(i); timeWriter.writeTimeMilli(dt.getMillisOfDay()); // Timestamp is milliseconds since the epoch, stored as 64-bit integer timeStampMilliWriter.setPosition(i); - timeStampMilliWriter.writeTimeStampMilli(dt.getMillis()); + timeStampMilliWriter.writeTimeStampMilli(org.apache.arrow.vector.util.DateUtility.toMillis(dt)); } writer.setValueCount(count); } @@ -176,13 +176,13 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) { printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { long dateVal = ((NullableDateMilliVector)root.getVector("date")).getAccessor().get(i); - DateTime dt = makeDateTimeFromCount(i); - DateTime dateExpected = dt.minusMillis(dt.getMillisOfDay()); - Assert.assertEquals(dateExpected.getMillis(), dateVal); + LocalDateTime dt = makeDateTimeFromCount(i); + LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay()); + Assert.assertEquals(org.apache.arrow.vector.util.DateUtility.toMillis(dateExpected), dateVal); long timeVal = ((NullableTimeMilliVector)root.getVector("time")).getAccessor().get(i); Assert.assertEquals(dt.getMillisOfDay(), timeVal); Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i); - Assert.assertTrue(dt.withZoneRetainFields(DateTimeZone.getDefault()).equals(timestampMilliVal)); + Assert.assertEquals(dt, timestampMilliVal); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java index 824c62aa5fb..f9c8f726ab6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java @@ -81,7 +81,7 @@ public void nestedSchema() { new Field("child4.1", true, Utf8.INSTANCE, null) ))); childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.of( - new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null), + new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, null), null), new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.of()) ))); Schema initialSchema = new Schema(childrenBuilder.build());