From bf245ce8e41d6d3469d65395f67895322651dbdd Mon Sep 17 00:00:00 2001 From: Julien Le Dem Date: Fri, 5 May 2017 19:01:52 -0700 Subject: [PATCH 1/4] add TZ vectors Change-Id: Iad4f080581382fa95a76cb5f64204111ead09a3d --- .../main/codegen/data/ValueVectorTypes.tdd | 24 +++++++++---- .../templates/AbstractFieldWriter.java | 10 +++--- .../AbstractPromotableFieldWriter.java | 31 +++++++---------- .../main/codegen/templates/BaseWriter.java | 4 +-- .../main/codegen/templates/ComplexCopier.java | 7 ++-- .../main/codegen/templates/MapWriters.java | 34 ++++++++++++++----- .../codegen/templates/UnionListWriter.java | 2 +- .../main/codegen/templates/UnionWriter.java | 6 ++-- .../org/apache/arrow/vector/types/Types.java | 28 +++++++++------ .../apache/arrow/vector/pojo/TestConvert.java | 3 +- 10 files changed, 90 insertions(+), 59 deletions(-) diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd index ca6d9ecbe85..cd88eb1c316 100644 --- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd +++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd @@ -71,12 +71,16 @@ minor: [ { class: "BigInt"}, { class: "UInt8" }, - { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}], }, - { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" }, - { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, - { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, - { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, - { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "Float8", javaType: "double", boxedType: "Double", fields: [{name: "value", type: "double"}] }, + { class: "DateMilli", javaType: "long", friendlyType: "LocalDateTime" }, + { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampMilliTZ", javaType: "long", boxedType: "Long", + typeParams: [ {name: "timezone", type: "String"} ], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp", + arrowTypeConstructorPrefix: "org.apache.arrow.vector.types.TimeUnit.MILLISECOND" }, { class: "TimeMicro" }, { class: "TimeNano" } ] @@ -97,7 +101,13 @@ boxedType: "ArrowBuf", minor: [ - { class: "Decimal", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] } + { + class: "Decimal", + maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", + typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal", + fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] + } ] }, { diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java index de076fc46ff..65bcc052ed4 100644 --- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java @@ -59,7 +59,7 @@ public void write(${name}Holder holder) { } <#if minor.class == "Decimal"> - public void writeDecimal(int start, ArrowBuf buffer) { + public void write${minor.class}(int start, ArrowBuf buffer) { fail("${name}"); } <#else> @@ -114,9 +114,11 @@ public ListWriter list(String name) { <#if lowerName == "int" ><#assign lowerName = "integer" /> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - <#if minor.class?starts_with("Decimal") > - public ${capName}Writer ${lowerName}(String name, int scale, int precision) { - fail("${capName}"); + <#if minor.typeParams?? > + + @Override + public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { + fail("${capName}(" + <#list minor.typeParams as typeParam>"${typeParam.name}: " + ${typeParam.name} + ", " + ")"); return null; } diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java index ada0b1d5c78..636b305d6db 100644 --- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -72,31 +72,23 @@ public void endList() { } <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> - <#assign fields = minor.fields!type.fields /> - <#if !minor.class?starts_with("Decimal") > + <#assign fields = minor.fields!type.fields /> @Override public void write(${name}Holder holder) { getWriter(MinorType.${name?upper_case}).write(holder); } + <#if minor.class == "Decimal"> + public void write${minor.class}(int start, ArrowBuf buffer) { + getWriter(MinorType.${name?upper_case}).write${minor.class}(start, buffer); + } + <#else> public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, ) { getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, ); } - - <#else> - @Override - public void write(DecimalHolder holder) { - getWriter(MinorType.DECIMAL).write(holder); - } - - public void writeDecimal(int start, ArrowBuf buffer) { - getWriter(MinorType.DECIMAL).writeDecimal(start, buffer); - } - - + - public void writeNull() { } @@ -125,10 +117,13 @@ public ListWriter list(String name) { <#if lowerName == "int" ><#assign lowerName = "integer" /> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - <#if minor.class?starts_with("Decimal") > - public ${capName}Writer ${lowerName}(String name, int scale, int precision) { - return getWriter(MinorType.MAP).${lowerName}(name, scale, precision); + + <#if minor.typeParams?? > + @Override + public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { + return getWriter(MinorType.MAP).${lowerName}(name<#list minor.typeParams as typeParam>, ${typeParam.name}); } + @Override public ${capName}Writer ${lowerName}(String name) { diff --git a/java/vector/src/main/codegen/templates/BaseWriter.java b/java/vector/src/main/codegen/templates/BaseWriter.java index 3da02b00a0d..405f466cbc7 100644 --- a/java/vector/src/main/codegen/templates/BaseWriter.java +++ b/java/vector/src/main/codegen/templates/BaseWriter.java @@ -53,8 +53,8 @@ public interface MapWriter extends BaseWriter { <#if lowerName == "int" ><#assign lowerName = "integer" /> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - <#if minor.class?starts_with("Decimal") > - ${capName}Writer ${lowerName}(String name, int scale, int precision); + <#if minor.typeParams?? > + ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}); ${capName}Writer ${lowerName}(String name); diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java index fb7ae0f2ef5..518ad5d398d 100644 --- a/java/vector/src/main/codegen/templates/ComplexCopier.java +++ b/java/vector/src/main/codegen/templates/ComplexCopier.java @@ -72,7 +72,8 @@ private static void writeValue(FieldReader reader, FieldWriter writer) { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + + <#if !minor.typeParams?? > case ${name?upper_case}: if (reader.isSet()) { @@ -94,7 +95,7 @@ private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter w <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams?? > case ${name?upper_case}: return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}(name); @@ -113,7 +114,7 @@ private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams?? > case ${name?upper_case}: return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}(); diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java index d3e6de95271..eea7311c2dc 100644 --- a/java/vector/src/main/codegen/templates/MapWriters.java +++ b/java/vector/src/main/codegen/templates/MapWriters.java @@ -56,7 +56,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter { this.container = container; for (Field child : container.getField().getChildren()) { - switch (Types.getMinorTypeForArrowType(child.getType())) { + MinorType minorType = Types.getMinorTypeForArrowType(child.getType()); + switch (minorType) { case MAP: map(child.getName()); break; @@ -71,15 +72,18 @@ public class ${mode}MapWriter extends AbstractFieldWriter { <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /> <#assign upperName = minor.class?upper_case /> - case ${upperName}: - <#if lowerName == "decimal" > - Decimal decimal = (Decimal)child.getType(); - decimal(child.getName(), decimal.getScale(), decimal.getPrecision()); + case ${upperName}: { + <#if minor.typeParams?? > + ${minor.arrowType} arrowType = (${minor.arrowType})child.getType(); + ${lowerName}(child.getName()<#list minor.typeParams as typeParam>, arrowType.get${typeParam.name?cap_first}()); <#else> ${lowerName}(child.getName()); - + break; + } + default: + throw new UnsupportedOperationException("Unknown type: " + minorType); } } } @@ -205,7 +209,8 @@ public void end() { <#assign vectName = capName /> <#assign vectName = "Nullable${capName}" /> - <#if minor.class?starts_with("Decimal") > + <#if minor.typeParams?? > + @Override public ${minor.class}Writer ${lowerName}(String name) { // returns existing writer final FieldWriter writer = fields.get(handleCase(name)); @@ -213,7 +218,8 @@ public void end() { return writer; } - public ${minor.class}Writer ${lowerName}(String name, int scale, int precision) { + @Override + public ${minor.class}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { <#else> @Override public ${minor.class}Writer ${lowerName}(String name) { @@ -223,7 +229,17 @@ public void end() { ValueVector vector; ValueVector currentVector = container.getChild(name); ${vectName}Vector v = container.addOrGet(name, - FieldType.nullable(<#if minor.class == "Decimal">new Decimal(precision, scale)<#else>MinorType.${upperName}.getType()), + FieldType.nullable( + <#if minor.arrowType??> + <#assign first = true /> + new ${minor.arrowType}(<#if minor.arrowTypeConstructorPrefix??> + ${minor.arrowTypeConstructorPrefix}, <#if minor.typeParams??><#list minor.typeParams as typeParam><#if first == true><#assign first = false /><#else>, + ${typeParam.name} + ) + <#else> + MinorType.${upperName}.getType() + + ), ${vectName}Vector.class); writer = new PromotableWriter(v, container, getNullableMapWriterFactory()); vector = v; diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java index d980830923b..d019a1edbb4 100644 --- a/java/vector/src/main/codegen/templates/UnionListWriter.java +++ b/java/vector/src/main/codegen/templates/UnionListWriter.java @@ -94,7 +94,7 @@ public void setPosition(int index) { <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams?? > @Override public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}() { diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java index 880f537c029..4a7c47245cf 100644 --- a/java/vector/src/main/codegen/templates/UnionWriter.java +++ b/java/vector/src/main/codegen/templates/UnionWriter.java @@ -121,7 +121,7 @@ BaseWriter getWriter(MinorType minorType) { <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams??> case ${name?upper_case}: return get${name}Writer(); @@ -136,7 +136,7 @@ BaseWriter getWriter(MinorType minorType) { <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams?? > private ${name}Writer ${name?uncap_first}Writer; @@ -206,7 +206,7 @@ public MapWriter map(String name) { <#if lowerName == "int" ><#assign lowerName = "integer" /> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - <#if !minor.class?starts_with("Decimal")> + <#if !minor.typeParams?? > @Override public ${capName}Writer ${lowerName}(String name) { data.getMutator().setType(idx(), MinorType.MAP); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 2a0e47b7bf7..f312a8c5c9f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -39,6 +39,7 @@ import org.apache.arrow.vector.NullableTimeNanoVector; import org.apache.arrow.vector.NullableTimeSecVector; import org.apache.arrow.vector.NullableTimeStampMicroVector; +import org.apache.arrow.vector.NullableTimeStampMilliTZVector; import org.apache.arrow.vector.NullableTimeStampMilliVector; import org.apache.arrow.vector.NullableTimeStampNanoVector; import org.apache.arrow.vector.NullableTimeStampSecVector; @@ -72,6 +73,7 @@ import org.apache.arrow.vector.complex.impl.TimeNanoWriterImpl; import org.apache.arrow.vector.complex.impl.TimeSecWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampMicroWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampMilliTZWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampMilliWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampNanoWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampSecWriterImpl; @@ -368,11 +370,6 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { } }, DECIMAL(null) { - @Override - public ArrowType getType() { - throw new UnsupportedOperationException("Cannot get simple type for Decimal type"); - } - @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new NullableDecimalVector(name, fieldType, allocator); @@ -439,11 +436,6 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { } }, FIXED_SIZE_LIST(null) { - @Override - public ArrowType getType() { - throw new UnsupportedOperationException("Cannot get simple type for FixedSizeList type"); - } - @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { return new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback); @@ -467,6 +459,17 @@ public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocato public FieldWriter getNewFieldWriter(ValueVector vector) { return new UnionWriter((UnionVector) vector); } + }, + TIMESTAMPMILLITZ(null) { + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { + return new NullableTimeStampMilliTZVector(name, fieldType, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampMilliTZWriterImpl((NullableTimeStampMilliTZVector) vector); + } }; private final ArrowType type; @@ -475,7 +478,10 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { this.type = type; } - public ArrowType getType() { + public final ArrowType getType() { + if (type == null) { + throw new UnsupportedOperationException("Cannot get simple type for type " + name()); + } return type; } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java index 64f797030ff..e2dae293321 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java @@ -83,7 +83,8 @@ public void nestedSchema() { ))); childrenBuilder.add(new Field("child5", FieldType.nullable(new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } )), ImmutableList.of( new Field("child5.1", FieldType.nullable(new Timestamp(TimeUnit.MILLISECOND, null)), null), - new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.of()) + new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.of()), + new Field("child5.3", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null) ))); Schema initialSchema = new Schema(childrenBuilder.build()); run(initialSchema); From 39966aa5791113982ab2e6fd077372f59b450d95 Mon Sep 17 00:00:00 2001 From: Julien Le Dem Date: Mon, 8 May 2017 12:14:21 -0700 Subject: [PATCH 2/4] add other vectors and tests Change-Id: I6960a9725489d4e9e396ac352d37d731e48de75e --- .../main/codegen/data/ValueVectorTypes.tdd | 14 +- .../codegen/templates/FixedValueVectors.java | 50 ++-- .../main/codegen/templates/MapWriters.java | 16 +- .../templates/NullableValueVectors.java | 36 ++- .../templates/VariableLengthVectors.java | 47 ++-- .../org/apache/arrow/vector/types/Types.java | 51 +++- .../complex/writer/TestComplexWriter.java | 232 +++++++++++++----- 7 files changed, 313 insertions(+), 133 deletions(-) diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd index cd88eb1c316..4d8c99e652f 100644 --- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd +++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd @@ -77,10 +77,22 @@ { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "LocalDateTime" }, + { class: "TimeStampSecTZ", javaType: "long", boxedType: "Long", + typeParams: [ {name: "timezone", type: "String"} ], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp", + arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.SECOND", "timezone"] }, { class: "TimeStampMilliTZ", javaType: "long", boxedType: "Long", typeParams: [ {name: "timezone", type: "String"} ], arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp", - arrowTypeConstructorPrefix: "org.apache.arrow.vector.types.TimeUnit.MILLISECOND" }, + arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.MILLISECOND", "timezone"] }, + { class: "TimeStampMicroTZ", javaType: "long", boxedType: "Long", + typeParams: [ {name: "timezone", type: "String"} ], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp", + arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.MICROSECOND", "timezone"] }, + { class: "TimeStampNanoTZ", javaType: "long", boxedType: "Long", + typeParams: [ {name: "timezone", type: "String"} ], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Timestamp", + arrowTypeConstructorParams: ["org.apache.arrow.vector.types.TimeUnit.NANOSECOND", "timezone"] }, { class: "TimeMicro" }, { class: "TimeNano" } ] diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java index f403ecfac1f..5d80b66b533 100644 --- a/java/vector/src/main/codegen/templates/FixedValueVectors.java +++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java @@ -25,9 +25,10 @@ <#list vv.types as type> <#list type.minor as minor> <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> +<#assign className = "${minor.class}Vector" /> <#if type.major == "Fixed"> -<@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" /> +<@pp.changeOutputFile name="/org/apache/arrow/vector/${className}.java" /> <#include "/@includes/license.ftl" /> package org.apache.arrow.vector; @@ -43,8 +44,8 @@ * * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{ - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); +public final class ${className} extends BaseDataValueVector implements FixedWidthVector{ + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); public static final int TYPE_WIDTH = ${type.width}; @@ -53,24 +54,25 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width}; private int allocationMonitor = 0; + <#if minor.typeParams??> - <#if minor.class == "Decimal"> + <#list minor.typeParams as typeParam> + private final ${typeParam.type} ${typeParam.name}; + - private int precision; - private int scale; - - public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) { + public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { super(name, allocator); - this.precision = precision; - this.scale = scale; + <#list minor.typeParams as typeParam> + this.${typeParam.name} = ${typeParam.name}; + } <#else> - public ${minor.class}Vector(String name, BufferAllocator allocator) { + + public ${className}(String name, BufferAllocator allocator) { super(name, allocator); } - @Override public MinorType getMinorType() { return MinorType.${minor.class?upper_case}; @@ -219,17 +221,17 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator){ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((${minor.class}Vector) to); + return new TransferImpl((${className}) to); } - public void transferTo(${minor.class}Vector target){ + public void transferTo(${className} target){ target.clear(); target.data = data.transferOwnership(target.allocator).buffer; target.data.writerIndex(data.writerIndex()); clear(); } - public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) { + public void splitAndTransferTo(int startIndex, int length, ${className} target) { final int startPoint = startIndex * ${type.width}; final int sliceLength = length * ${type.width}; target.clear(); @@ -238,22 +240,18 @@ public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector } private class TransferImpl implements TransferPair{ - private ${minor.class}Vector to; + private ${className} to; public TransferImpl(String name, BufferAllocator allocator){ - <#if minor.class == "Decimal"> - to = new ${minor.class}Vector(name, allocator, precision, scale); - <#else> - to = new ${minor.class}Vector(name, allocator); - + to = new ${className}(name, allocator<#if minor.typeParams??><#list minor.typeParams as typeParam>, ${className}.this.${typeParam.name}); } - public TransferImpl(${minor.class}Vector to) { + public TransferImpl(${className} to) { this.to = to; } @Override - public ${minor.class}Vector getTo(){ + public ${className} getTo(){ return to; } @@ -269,11 +267,11 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this); + to.copyFromSafe(fromIndex, toIndex, ${className}.this); } } - public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFrom(int fromIndex, int thisIndex, ${className} from){ <#if (type.width > 8 || minor.class == "IntervalDay")> from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width}); <#else> <#-- type.width <= 8 --> @@ -283,7 +281,7 @@ public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ <#-- type.width --> } - public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){ while(thisIndex >= getValueCapacity()) { reAlloc(); } diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java index eea7311c2dc..05048c5c341 100644 --- a/java/vector/src/main/codegen/templates/MapWriters.java +++ b/java/vector/src/main/codegen/templates/MapWriters.java @@ -230,12 +230,16 @@ public void end() { ValueVector currentVector = container.getChild(name); ${vectName}Vector v = container.addOrGet(name, FieldType.nullable( - <#if minor.arrowType??> - <#assign first = true /> - new ${minor.arrowType}(<#if minor.arrowTypeConstructorPrefix??> - ${minor.arrowTypeConstructorPrefix}, <#if minor.typeParams??><#list minor.typeParams as typeParam><#if first == true><#assign first = false /><#else>, - ${typeParam.name} - ) + <#if minor.typeParams??> + <#if minor.arrowTypeConstructorParams??> + <#assign constructorParams = minor.arrowTypeConstructorParams /> + <#else> + <#assign constructorParams = [] /> + <#list minor.typeParams as typeParam> + <#assign constructorParams = constructorParams + [ typeParam.name ] /> + + + new ${minor.arrowType}(${constructorParams?join(", ")}) <#else> MinorType.${upperName}.getType() diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java index ed2418e4a2a..3231c4c6e0a 100644 --- a/java/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java @@ -61,12 +61,27 @@ public final class ${className} extends BaseDataValueVector implements <#if type private final List innerVectors; - <#if minor.class == "Decimal"> - private final int precision; - private final int scale; + <#if minor.typeParams??> + <#list minor.typeParams as typeParam> + private final ${typeParam.type} ${typeParam.name}; + - public ${className}(String name, BufferAllocator allocator, int precision, int scale) { - this(name, FieldType.nullable(new Decimal(precision, scale)), allocator); + /** + * Assumes the type is nullable and not dictionary encoded + * @param name name of the field + * @param allocator allocator to use to resize the vector<#list minor.typeParams as typeParam> + * @param ${typeParam.name} type parameter ${typeParam.name} + */ + public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { + <#if minor.arrowTypeConstructorParams??> + <#assign constructorParams = minor.arrowTypeConstructorParams /> + <#else> + <#assign constructorParams = [] /> + <#list minor.typeParams as typeParam> + <#assign constructorParams = constructorParams + [ typeParam.name ] /> + + + this(name, FieldType.nullable(new ${minor.arrowType}(${constructorParams?join(", ")})), allocator); } <#else> public ${className}(String name, BufferAllocator allocator) { @@ -76,11 +91,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type public ${className}(String name, FieldType fieldType, BufferAllocator allocator) { super(name, allocator); - <#if minor.class == "Decimal"> - Decimal decimal = (Decimal)fieldType.getType(); - this.precision = decimal.getPrecision(); - this.scale = decimal.getScale(); - this.values = new ${valuesName}(valuesField, allocator, precision, scale); + <#if minor.typeParams??> + ${minor.arrowType} arrowType = (${minor.arrowType})fieldType.getType(); + <#list minor.typeParams as typeParam> + this.${typeParam.name} = arrowType.get${typeParam.name?cap_first}(); + + this.values = new ${valuesName}(valuesField, allocator<#list minor.typeParams as typeParam>, ${typeParam.name}); <#else> this.values = new ${valuesName}(valuesField, allocator); diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java index 3d933addb62..f13291b79ad 100644 --- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java +++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java @@ -28,6 +28,7 @@ <#list type.minor as minor> <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> +<#assign className = "${minor.class}Vector" /> <#if type.major == "VarLen"> <@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" /> @@ -48,8 +49,8 @@ * * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{ - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); +public final class ${className} extends BaseDataValueVector implements VariableWidthVector{ + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); private static final int DEFAULT_RECORD_BYTE_COUNT = 8; private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT; @@ -66,22 +67,22 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V private int allocationSizeInBytes = INITIAL_BYTE_COUNT; private int allocationMonitor = 0; - <#if minor.class == "Decimal"> + <#if minor.typeParams??> + <#list minor.typeParams as typeParam> + private final ${typeParam.type} ${typeParam.name}; + - private final int precision; - private final int scale; - - public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) { + public ${className}(String name, BufferAllocator allocator<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}) { super(name, allocator); this.oAccessor = offsetVector.getAccessor(); this.accessor = new Accessor(); this.mutator = new Mutator(); - this.precision = precision; - this.scale = scale; + <#list minor.typeParams as typeParam> + this.${typeParam.name} = ${typeParam.name}; + } <#else> - - public ${minor.class}Vector(String name, BufferAllocator allocator) { + public ${className}(String name, BufferAllocator allocator) { super(name, allocator); this.oAccessor = offsetVector.getAccessor(); this.accessor = new Accessor(); @@ -188,10 +189,10 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator){ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((${minor.class}Vector) to); + return new TransferImpl((${className}) to); } - public void transferTo(${minor.class}Vector target){ + public void transferTo(${className} target){ target.clear(); this.offsetVector.transferTo(target.offsetVector); target.data = data.transferOwnership(target.allocator).buffer; @@ -199,7 +200,7 @@ public void transferTo(${minor.class}Vector target){ clear(); } - public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) { + public void splitAndTransferTo(int startIndex, int length, ${className} target) { UInt${type.width}Vector.Accessor offsetVectorAccessor = this.offsetVector.getAccessor(); final int startPoint = offsetVectorAccessor.get(startIndex); final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint; @@ -214,7 +215,7 @@ public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target.getMutator().setValueCount(length); } - protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ + protected void copyFrom(int fromIndex, int thisIndex, ${className} from){ final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor(); final int start = fromOffsetVectorAccessor.get(fromIndex); final int end = fromOffsetVectorAccessor.get(fromIndex + 1); @@ -225,7 +226,7 @@ protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len); } - public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public boolean copyFromSafe(int fromIndex, int thisIndex, ${className} from){ final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor(); final int start = fromOffsetVectorAccessor.get(fromIndex); final int end = fromOffsetVectorAccessor.get(fromIndex + 1); @@ -242,22 +243,18 @@ public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector f } private class TransferImpl implements TransferPair{ - ${minor.class}Vector to; + ${className} to; public TransferImpl(String name, BufferAllocator allocator){ - <#if minor.class == "Decimal"> - to = new ${minor.class}Vector(name, allocator, precision, scale); - <#else> - to = new ${minor.class}Vector(name, allocator); - + to = new ${className}(name, allocator<#if minor.typeParams??><#list minor.typeParams as typeParam>, ${className}.this.${typeParam.name}); } - public TransferImpl(${minor.class}Vector to){ + public TransferImpl(${className} to){ this.to = to; } @Override - public ${minor.class}Vector getTo(){ + public ${className} getTo(){ return to; } @@ -273,7 +270,7 @@ public void splitAndTransfer(int startIndex, int length) { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this); + to.copyFromSafe(fromIndex, toIndex, ${className}.this); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index f312a8c5c9f..6591a4b16da 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -38,10 +38,13 @@ import org.apache.arrow.vector.NullableTimeMilliVector; import org.apache.arrow.vector.NullableTimeNanoVector; import org.apache.arrow.vector.NullableTimeSecVector; +import org.apache.arrow.vector.NullableTimeStampMicroTZVector; import org.apache.arrow.vector.NullableTimeStampMicroVector; import org.apache.arrow.vector.NullableTimeStampMilliTZVector; import org.apache.arrow.vector.NullableTimeStampMilliVector; +import org.apache.arrow.vector.NullableTimeStampNanoTZVector; import org.apache.arrow.vector.NullableTimeStampNanoVector; +import org.apache.arrow.vector.NullableTimeStampSecTZVector; import org.apache.arrow.vector.NullableTimeStampSecVector; import org.apache.arrow.vector.NullableTinyIntVector; import org.apache.arrow.vector.NullableUInt1Vector; @@ -72,10 +75,13 @@ import org.apache.arrow.vector.complex.impl.TimeMilliWriterImpl; import org.apache.arrow.vector.complex.impl.TimeNanoWriterImpl; import org.apache.arrow.vector.complex.impl.TimeSecWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampMicroTZWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampMicroWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampMilliTZWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampMilliWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampNanoTZWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampNanoWriterImpl; +import org.apache.arrow.vector.complex.impl.TimeStampSecTZWriterImpl; import org.apache.arrow.vector.complex.impl.TimeStampSecWriterImpl; import org.apache.arrow.vector.complex.impl.TinyIntWriterImpl; import org.apache.arrow.vector.complex.impl.UInt1WriterImpl; @@ -460,6 +466,17 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { return new UnionWriter((UnionVector) vector); } }, + TIMESTAMPSECTZ(null) { + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { + return new NullableTimeStampSecTZVector(name, fieldType, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampSecTZWriterImpl((NullableTimeStampSecTZVector) vector); + } + }, TIMESTAMPMILLITZ(null) { @Override public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { @@ -470,6 +487,28 @@ public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocato public FieldWriter getNewFieldWriter(ValueVector vector) { return new TimeStampMilliTZWriterImpl((NullableTimeStampMilliTZVector) vector); } + }, + TIMESTAMPMICROTZ(null) { + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { + return new NullableTimeStampMicroTZVector(name, fieldType, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampMicroTZWriterImpl((NullableTimeStampMicroTZVector) vector); + } + }, + TIMESTAMPNANOTZ(null) { + @Override + public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) { + return new NullableTimeStampNanoTZVector(name, fieldType, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new TimeStampNanoTZWriterImpl((NullableTimeStampNanoTZVector) vector); + } }; private final ArrowType type; @@ -585,18 +624,16 @@ public MinorType visit(FloatingPoint type) { } @Override public MinorType visit(Timestamp type) { - if (type.getTimezone() != null) { - throw new IllegalArgumentException("only timezone-less timestamps are supported for now: " + type); - } + String tz = type.getTimezone(); switch (type.getUnit()) { case SECOND: - return MinorType.TIMESTAMPSEC; + return tz == null ? MinorType.TIMESTAMPSEC : MinorType.TIMESTAMPSECTZ; case MILLISECOND: - return MinorType.TIMESTAMPMILLI; + return tz == null ? MinorType.TIMESTAMPMILLI : MinorType.TIMESTAMPMILLITZ; case MICROSECOND: - return MinorType.TIMESTAMPMICRO; + return tz == null ? MinorType.TIMESTAMPMICRO : MinorType.TIMESTAMPMICROTZ; case NANOSECOND: - return MinorType.TIMESTAMPNANO; + return tz == null ? MinorType.TIMESTAMPNANO : MinorType.TIMESTAMPNANOTZ; default: throw new IllegalArgumentException("unknown unit: " + type); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 16139361756..ede8d65f075 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -42,15 +42,18 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter; import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter; +import org.apache.arrow.vector.holders.NullableTimeStampNanoTZHolder; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID; import org.apache.arrow.vector.types.pojo.ArrowType.Int; import org.apache.arrow.vector.types.pojo.ArrowType.Struct; +import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp; import org.apache.arrow.vector.types.pojo.ArrowType.Union; import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; +import org.apache.arrow.vector.util.DateUtility; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.Text; @@ -592,85 +595,198 @@ public void mapWriterMixedCaseFieldNames() { } @Test - public void timeStampWriters() throws Exception { + public void timeStampSecWriter() throws Exception { // test values - final long expectedNanos = 981173106123456789L; - final long expectedMicros = 981173106123456L; - final long expectedMillis = 981173106123L; final long expectedSecs = 981173106L; final LocalDateTime expectedSecDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 0); + + // write + MapVector parent = new MapVector("parent", allocator, null); + ComplexWriter writer = new ComplexWriterImpl("root", parent); + MapWriter rootWriter = writer.rootAsMap(); + + { + TimeStampSecWriter timeStampSecWriter = rootWriter.timeStampSec("sec"); + timeStampSecWriter.setPosition(0); + timeStampSecWriter.writeTimeStampSec(expectedSecs); + } + { + TimeStampSecTZWriter timeStampSecTZWriter = rootWriter.timeStampSecTZ("secTZ", "UTC"); + timeStampSecTZWriter.setPosition(1); + timeStampSecTZWriter.writeTimeStampSecTZ(expectedSecs); + } + // schema + List children = parent.getField().getChildren().get(0).getChildren(); + checkTimestampField(children.get(0), "sec"); + checkTimestampTZField(children.get(1), "secTZ", "UTC"); + + // read + MapReader rootReader = new SingleMapReaderImpl(parent).reader("root"); + { + FieldReader secReader = rootReader.reader("sec"); + secReader.setPosition(0); + LocalDateTime secDateTime = secReader.readLocalDateTime(); + Assert.assertEquals(expectedSecDateTime, secDateTime); + long secLong = secReader.readLong(); + Assert.assertEquals(expectedSecs, secLong); + } + { + FieldReader secTZReader = rootReader.reader("secTZ"); + secTZReader.setPosition(1); + long secTZLong = secTZReader.readLong(); + Assert.assertEquals(expectedSecs, secTZLong); + } + } + + @Test + public void timeStampMilliWriters() throws Exception { + // test values + final long expectedMillis = 981173106123L; final LocalDateTime expectedMilliDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123); - final LocalDateTime expectedMicroDateTime = expectedMilliDateTime; - final LocalDateTime expectedNanoDateTime = expectedMilliDateTime; // write MapVector parent = MapVector.empty("parent", allocator); ComplexWriter writer = new ComplexWriterImpl("root", parent); MapWriter rootWriter = writer.rootAsMap(); + { + TimeStampMilliWriter timeStampWriter = rootWriter.timeStampMilli("milli"); + timeStampWriter.setPosition(0); + timeStampWriter.writeTimeStampMilli(expectedMillis); + } + String tz = DateUtility.getTimeZone(10); + { + TimeStampMilliTZWriter timeStampTZWriter = rootWriter.timeStampMilliTZ("milliTZ", tz); + timeStampTZWriter.setPosition(0); + timeStampTZWriter.writeTimeStampMilliTZ(expectedMillis); + } + // schema + List children = parent.getField().getChildren().get(0).getChildren(); + checkTimestampField(children.get(0), "milli"); + checkTimestampTZField(children.get(1), "milliTZ", tz); + + // read + MapReader rootReader = new SingleMapReaderImpl(parent).reader("root"); + + { + FieldReader milliReader = rootReader.reader("milli"); + milliReader.setPosition(0); + LocalDateTime milliDateTime = milliReader.readLocalDateTime(); + Assert.assertEquals(expectedMilliDateTime, milliDateTime); + long milliLong = milliReader.readLong(); + Assert.assertEquals(expectedMillis, milliLong); + } + { + FieldReader milliTZReader = rootReader.reader("milliTZ"); + milliTZReader.setPosition(0); + long milliTZLong = milliTZReader.readLong(); + Assert.assertEquals(expectedMillis, milliTZLong); + } - TimeStampSecWriter timeStampSecWriter = rootWriter.timeStampSec("sec"); - timeStampSecWriter.setPosition(0); - timeStampSecWriter.writeTimeStampSec(expectedSecs); + } - TimeStampMilliWriter timeStampWriter = rootWriter.timeStampMilli("milli"); - timeStampWriter.setPosition(1); - timeStampWriter.writeTimeStampMilli(expectedMillis); + private void checkTimestampField(Field field, String name) { + Assert.assertEquals(name, field.getName()); + Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, field.getType().getTypeID()); + } - TimeStampMicroWriter timeStampMicroWriter = rootWriter.timeStampMicro("micro"); - timeStampMicroWriter.setPosition(2); - timeStampMicroWriter.writeTimeStampMicro(expectedMicros); + private void checkTimestampTZField(Field field, String name, String tz) { + checkTimestampField(field, name); + Assert.assertEquals(tz, ((Timestamp)field.getType()).getTimezone()); + } - TimeStampNanoWriter timeStampNanoWriter = rootWriter.timeStampNano("nano"); - timeStampNanoWriter.setPosition(3); - timeStampNanoWriter.writeTimeStampNano(expectedNanos); + @Test + public void timeStampMicroWriters() throws Exception { + // test values + final long expectedMicros = 981173106123456L; + final LocalDateTime expectedMicroDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123); + + // write + MapVector parent = new MapVector("parent", allocator, null); + ComplexWriter writer = new ComplexWriterImpl("root", parent); + MapWriter rootWriter = writer.rootAsMap(); + + { + TimeStampMicroWriter timeStampMicroWriter = rootWriter.timeStampMicro("micro"); + timeStampMicroWriter.setPosition(0); + timeStampMicroWriter.writeTimeStampMicro(expectedMicros); + } + String tz = DateUtility.getTimeZone(5); + { + TimeStampMicroTZWriter timeStampMicroWriter = rootWriter.timeStampMicroTZ("microTZ", tz); + timeStampMicroWriter.setPosition(1); + timeStampMicroWriter.writeTimeStampMicroTZ(expectedMicros); + } // schema - Field secField = parent.getField().getChildren().get(0).getChildren().get(0); - Assert.assertEquals("sec", secField.getName()); - Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, secField.getType().getTypeID()); + List children = parent.getField().getChildren().get(0).getChildren(); + checkTimestampField(children.get(0), "micro"); + checkTimestampTZField(children.get(1), "microTZ", tz); + + // read + MapReader rootReader = new SingleMapReaderImpl(parent).reader("root"); + { + FieldReader microReader = rootReader.reader("micro"); + microReader.setPosition(0); + LocalDateTime microDateTime = microReader.readLocalDateTime(); + Assert.assertEquals(expectedMicroDateTime, microDateTime); + long microLong = microReader.readLong(); + Assert.assertEquals(expectedMicros, microLong); + } + { + FieldReader microReader = rootReader.reader("microTZ"); + microReader.setPosition(1); + long microLong = microReader.readLong(); + Assert.assertEquals(expectedMicros, microLong); + } - Field milliField = parent.getField().getChildren().get(0).getChildren().get(1); - Assert.assertEquals("milli", milliField.getName()); - Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, milliField.getType().getTypeID()); + } - Field microField = parent.getField().getChildren().get(0).getChildren().get(2); - Assert.assertEquals("micro", microField.getName()); - Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, microField.getType().getTypeID()); + @Test + public void timeStampNanoWriters() throws Exception { + // test values + final long expectedNanos = 981173106123456789L; + final LocalDateTime expectedNanoDateTime = new LocalDateTime(2001, 2, 3, 4, 5, 6, 123); - Field nanoField = parent.getField().getChildren().get(0).getChildren().get(3); - Assert.assertEquals("nano", nanoField.getName()); - Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, nanoField.getType().getTypeID()); + // write + MapVector parent = new MapVector("parent", allocator, null); + ComplexWriter writer = new ComplexWriterImpl("root", parent); + MapWriter rootWriter = writer.rootAsMap(); + { + TimeStampNanoWriter timeStampNanoWriter = rootWriter.timeStampNano("nano"); + timeStampNanoWriter.setPosition(0); + timeStampNanoWriter.writeTimeStampNano(expectedNanos); + } + String tz = DateUtility.getTimeZone(3); + { + TimeStampNanoTZWriter timeStampNanoWriter = rootWriter.timeStampNanoTZ("nanoTZ", tz); + timeStampNanoWriter.setPosition(0); + timeStampNanoWriter.writeTimeStampNanoTZ(expectedNanos); + } + // schema + List children = parent.getField().getChildren().get(0).getChildren(); + checkTimestampField(children.get(0), "nano"); + checkTimestampTZField(children.get(1), "nanoTZ", tz); // read MapReader rootReader = new SingleMapReaderImpl(parent).reader("root"); - FieldReader secReader = rootReader.reader("sec"); - secReader.setPosition(0); - LocalDateTime secDateTime = secReader.readLocalDateTime(); - Assert.assertEquals(expectedSecDateTime, secDateTime); - long secLong = secReader.readLong(); - Assert.assertEquals(expectedSecs, secLong); - - FieldReader milliReader = rootReader.reader("milli"); - milliReader.setPosition(1); - LocalDateTime milliDateTime = milliReader.readLocalDateTime(); - Assert.assertEquals(expectedMilliDateTime, milliDateTime); - long milliLong = milliReader.readLong(); - Assert.assertEquals(expectedMillis, milliLong); - - FieldReader microReader = rootReader.reader("micro"); - microReader.setPosition(2); - LocalDateTime microDateTime = microReader.readLocalDateTime(); - Assert.assertEquals(expectedMicroDateTime, microDateTime); - long microLong = microReader.readLong(); - Assert.assertEquals(expectedMicros, microLong); - - FieldReader nanoReader = rootReader.reader("nano"); - nanoReader.setPosition(3); - LocalDateTime nanoDateTime = nanoReader.readLocalDateTime(); - Assert.assertEquals(expectedNanoDateTime, nanoDateTime); - long nanoLong = nanoReader.readLong(); - Assert.assertEquals(expectedNanos, nanoLong); + { + FieldReader nanoReader = rootReader.reader("nano"); + nanoReader.setPosition(0); + LocalDateTime nanoDateTime = nanoReader.readLocalDateTime(); + Assert.assertEquals(expectedNanoDateTime, nanoDateTime); + long nanoLong = nanoReader.readLong(); + Assert.assertEquals(expectedNanos, nanoLong); + } + { + FieldReader nanoReader = rootReader.reader("nanoTZ"); + nanoReader.setPosition(0); + long nanoLong = nanoReader.readLong(); + Assert.assertEquals(expectedNanos, nanoLong); + NullableTimeStampNanoTZHolder h = new NullableTimeStampNanoTZHolder(); + nanoReader.read(h); + Assert.assertEquals(expectedNanos, h.value); + } } @Test @@ -710,4 +826,4 @@ public void complexCopierWithList() { innerMap = (JsonStringHashMap) object.get(3); assertEquals(2, innerMap.get("a")); } -} \ No newline at end of file +} From a58fcae6da0994d893042d67eee4e84f25a3fdda Mon Sep 17 00:00:00 2001 From: Julien Le Dem Date: Mon, 8 May 2017 13:24:28 -0700 Subject: [PATCH 3/4] add integration test Change-Id: Ibbe3453e3b627b5187c496d91d296e538dd3740d --- integration/integration_test.py | 4 +++- .../vector/file/json/JsonFileReader.java | 20 +++++++++++++++++-- .../arrow/vector/file/BaseFileTest.java | 13 +++++++++--- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/integration/integration_test.py b/integration/integration_test.py index 646646997f7..b652eb1f881 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -620,7 +620,9 @@ def generate_datetime_case(): TimestampType('f7', 'ms'), TimestampType('f8', 'us'), TimestampType('f9', 'ns'), - TimestampType('f10', 'ms', tz=None) + TimestampType('f10', 'ms', tz=None), + TimestampType('f11', 'ms', tz='UTC'), + TimestampType('f12', 'ms', tz='US/Eastern') ] batch_sizes = [7, 10] diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java index 21aa0372c6b..3ef1484e7ea 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java @@ -43,9 +43,13 @@ import org.apache.arrow.vector.TimeMilliVector; import org.apache.arrow.vector.TimeNanoVector; import org.apache.arrow.vector.TimeSecVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; import org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.arrow.vector.TimeStampMilliTZVector; import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampNanoTZVector; import org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.arrow.vector.TimeStampSecTZVector; import org.apache.arrow.vector.TimeStampSecVector; import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.UInt1Vector; @@ -61,14 +65,14 @@ import org.apache.arrow.vector.schema.ArrowVectorType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.MappingJsonFactory; import com.google.common.base.Objects; -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; public class JsonFileReader implements AutoCloseable { private final File inputFile; @@ -278,6 +282,18 @@ private void setValueFromParser(ValueVector valueVector, int i) throws IOExcepti case TIMESTAMPNANO: ((TimeStampNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); break; + case TIMESTAMPSECTZ: + ((TimeStampSecTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMILLITZ: + ((TimeStampMilliTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; + case TIMESTAMPMICROTZ: + ((TimeStampMicroTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; + case TIMESTAMPNANOTZ: + ((TimeStampNanoTZVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class)); + break; default: throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType()); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java index 5cc36a3b820..63027e6c463 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java @@ -37,8 +37,10 @@ import org.apache.arrow.vector.complex.writer.DateMilliWriter; import org.apache.arrow.vector.complex.writer.IntWriter; import org.apache.arrow.vector.complex.writer.TimeMilliWriter; +import org.apache.arrow.vector.complex.writer.TimeStampMilliTZWriter; import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter; import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder; +import org.apache.arrow.vector.util.DateUtility; import org.joda.time.DateTimeZone; import org.joda.time.LocalDateTime; import org.junit.After; @@ -155,18 +157,21 @@ protected void writeDateTimeData(int count, NullableMapVector parent) { DateMilliWriter dateWriter = rootWriter.dateMilli("date"); TimeMilliWriter timeWriter = rootWriter.timeMilli("time"); TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli"); + TimeStampMilliTZWriter timeStampMilliTZWriter = rootWriter.timeStampMilliTZ("timestamp-milliTZ", "Europe/Paris"); for (int i = 0; i < count; i++) { LocalDateTime dt = makeDateTimeFromCount(i); // Number of days in milliseconds since epoch, stored as 64-bit integer, only date part is used dateWriter.setPosition(i); - long dateLong = org.apache.arrow.vector.util.DateUtility.toMillis(dt.minusMillis(dt.getMillisOfDay())); + long dateLong = DateUtility.toMillis(dt.minusMillis(dt.getMillisOfDay())); dateWriter.writeDateMilli(dateLong); // Time is a value in milliseconds since midnight, stored as 32-bit integer timeWriter.setPosition(i); timeWriter.writeTimeMilli(dt.getMillisOfDay()); // Timestamp is milliseconds since the epoch, stored as 64-bit integer timeStampMilliWriter.setPosition(i); - timeStampMilliWriter.writeTimeStampMilli(org.apache.arrow.vector.util.DateUtility.toMillis(dt)); + timeStampMilliWriter.writeTimeStampMilli(DateUtility.toMillis(dt)); + timeStampMilliTZWriter.setPosition(i); + timeStampMilliTZWriter.writeTimeStampMilliTZ(DateUtility.toMillis(dt)); } writer.setValueCount(count); } @@ -178,11 +183,13 @@ protected void validateDateTimeContent(int count, VectorSchemaRoot root) { long dateVal = ((NullableDateMilliVector)root.getVector("date")).getAccessor().get(i); LocalDateTime dt = makeDateTimeFromCount(i); LocalDateTime dateExpected = dt.minusMillis(dt.getMillisOfDay()); - Assert.assertEquals(org.apache.arrow.vector.util.DateUtility.toMillis(dateExpected), dateVal); + Assert.assertEquals(DateUtility.toMillis(dateExpected), dateVal); long timeVal = ((NullableTimeMilliVector)root.getVector("time")).getAccessor().get(i); Assert.assertEquals(dt.getMillisOfDay(), timeVal); Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i); Assert.assertEquals(dt, timestampMilliVal); + Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getAccessor().getObject(i); + Assert.assertEquals(DateUtility.toMillis(dt), timestampMilliTZVal); } } From 37987b94fb851e1303d5cdc35591779f60f46fa7 Mon Sep 17 00:00:00 2001 From: Julien Le Dem Date: Mon, 8 May 2017 13:27:09 -0700 Subject: [PATCH 4/4] add integration tests Change-Id: I060ef92dbb002202cb964280a122a2f4f3ad6fdb --- integration/integration_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration/integration_test.py b/integration/integration_test.py index b652eb1f881..cc59593dbdc 100644 --- a/integration/integration_test.py +++ b/integration/integration_test.py @@ -621,8 +621,10 @@ def generate_datetime_case(): TimestampType('f8', 'us'), TimestampType('f9', 'ns'), TimestampType('f10', 'ms', tz=None), - TimestampType('f11', 'ms', tz='UTC'), - TimestampType('f12', 'ms', tz='US/Eastern') + TimestampType('f11', 's', tz='UTC'), + TimestampType('f12', 'ms', tz='US/Eastern'), + TimestampType('f13', 'us', tz='Europe/Paris'), + TimestampType('f14', 'ns', tz='US/Pacific') ] batch_sizes = [7, 10]