Skip to content

ARROW-315: finalize timestamp #156

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
julienledem wants to merge 2 commits into from
Closed

ARROW-315: finalize timestamp #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
86cae98
ARROW-315: finalize timestamp
julienledem Oct 3, 2016
0ee017f
review feedback
julienledem Oct 4, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion format/Message.fbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,11 @@ table Date {
table Time {
}

enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND }

Copy link
Member

@wesm wesm Oct 4, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the thought that a time zone (if any) would go in the custom metadata?

Copy link
Member Author

@julienledem julienledem Oct 4, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Timezoned data should be converted to UTC before putting in arrow.
That's simpler.
We can add timezone later on if necessary.

Copy link
Member

@wesm wesm Oct 4, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's fine with me. My experience has been that having both time zone naive and non-naive timestamps is more trouble than it's worth

/// time from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
table Timestamp {
timezone: string;
unit: TimeUnit;
}

enum IntervalUnit: short { YEAR_MONTH, DAY_TIME}
Expand Down
2 changes: 1 addition & 1 deletion java/vector/src/main/codegen/data/ArrowTypes.tdd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
},
{
name: "Timestamp",
fields: [{name: "timezone", type: "String"}]
fields: [{name: "unit", type: short}]
},
{
name: "Interval",
Expand Down
2 changes: 1 addition & 1 deletion java/vector/src/main/codegen/templates/NullableValueVectors.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#elseif minor.class == "Float8">
field = new Field(name, true, new FloatingPoint(Precision.DOUBLE), null);
<#elseif minor.class == "TimeStamp">
field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(""), null);
field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND), null);
<#elseif minor.class == "IntervalDay">
field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.DAY_TIME), null);
<#elseif minor.class == "IntervalYear">
Expand Down
46 changes: 26 additions & 20 deletions java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import org.apache.arrow.flatbuf.IntervalUnit;
import org.apache.arrow.flatbuf.Precision;
import org.apache.arrow.flatbuf.TimeUnit;
import org.apache.arrow.flatbuf.UnionMode;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.FieldVector;
Expand Down Expand Up @@ -101,7 +102,7 @@ public class Types {
private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
private static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(""), null);
private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND), null);
private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(Precision.SINGLE), null);
Expand Down Expand Up @@ -143,8 +144,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new NullableMapWriter((NullableMapVector) vector);
}
}, // an empty map column. Useful for conceptual setup. Children listed within here

},
TINYINT(new Int(8, true)) {
@Override
public Field getField() {
Expand All @@ -160,7 +160,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new TinyIntWriterImpl((NullableTinyIntVector) vector);
}
}, // single byte signed integer
},
SMALLINT(new Int(16, true)) {
@Override
public Field getField() {
Expand All @@ -176,7 +176,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new SmallIntWriterImpl((NullableSmallIntVector) vector);
}
}, // two byte signed integer
},
INT(new Int(32, true)) {
@Override
public Field getField() {
Expand All @@ -192,7 +192,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new IntWriterImpl((NullableIntVector) vector);
}
}, // four byte signed integer
},
BIGINT(new Int(64, true)) {
@Override
public Field getField() {
Expand All @@ -208,7 +208,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new BigIntWriterImpl((NullableBigIntVector) vector);
}
}, // eight byte signed integer
},
DATE(Date.INSTANCE) {
@Override
public Field getField() {
Expand All @@ -224,7 +224,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new DateWriterImpl((NullableDateVector) vector);
}
}, // days since 4713bc
},
TIME(Time.INSTANCE) {
@Override
public Field getField() {
Expand All @@ -240,8 +240,9 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new TimeWriterImpl((NullableTimeVector) vector);
}
}, // time in micros before or after 2000/1/1
TIMESTAMP(new Timestamp("")) {
},
// time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
TIMESTAMP(new Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND)) {
@Override
public Field getField() {
return TIMESTAMP_FIELD;
Expand Down Expand Up @@ -289,6 +290,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) {
return new IntervalYearWriterImpl((NullableIntervalYearVector) vector);
}
},
// 4 byte ieee 754
FLOAT4(new FloatingPoint(Precision.SINGLE)) {
@Override
public Field getField() {
Expand All @@ -304,7 +306,8 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new Float4WriterImpl((NullableFloat4Vector) vector);
}
}, // 4 byte ieee 754
},
// 8 byte ieee 754
FLOAT8(new FloatingPoint(Precision.DOUBLE)) {
@Override
public Field getField() {
Expand All @@ -320,7 +323,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new Float8WriterImpl((NullableFloat8Vector) vector);
}
}, // 8 byte ieee 754
},
BIT(Bool.INSTANCE) {
@Override
public Field getField() {
Expand All @@ -336,7 +339,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new BitWriterImpl((NullableBitVector) vector);
}
}, // single bit value (boolean)
},
VARCHAR(Utf8.INSTANCE) {
@Override
public Field getField() {
Expand All @@ -352,7 +355,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new VarCharWriterImpl((NullableVarCharVector) vector);
}
}, // utf8 variable length string
},
VARBINARY(Binary.INSTANCE) {
@Override
public Field getField() {
Expand All @@ -368,7 +371,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new VarBinaryWriterImpl((NullableVarBinaryVector) vector);
}
}, // variable length binary
},
DECIMAL(null) {
@Override
public ArrowType getType() {
Expand All @@ -388,7 +391,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new DecimalWriterImpl((NullableDecimalVector) vector);
}
}, // variable length binary
},
UINT1(new Int(8, false)) {
@Override
public Field getField() {
Expand All @@ -404,7 +407,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new UInt1WriterImpl((NullableUInt1Vector) vector);
}
}, // unsigned 1 byte integer
},
UINT2(new Int(16, false)) {
@Override
public Field getField() {
Expand All @@ -420,7 +423,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new UInt2WriterImpl((NullableUInt2Vector) vector);
}
}, // unsigned 2 byte integer
},
UINT4(new Int(32, false)) {
@Override
public Field getField() {
Expand All @@ -436,7 +439,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new UInt4WriterImpl((NullableUInt4Vector) vector);
}
}, // unsigned 4 byte integer
},
UINT8(new Int(64, false)) {
@Override
public Field getField() {
Expand All @@ -452,7 +455,7 @@ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack
public FieldWriter getNewFieldWriter(ValueVector vector) {
return new UInt8WriterImpl((NullableUInt8Vector) vector);
}
}, // unsigned 8 byte integer
},
LIST(List.INSTANCE) {
@Override
public Field getField() {
Expand Down Expand Up @@ -576,6 +579,9 @@ public MinorType visit(FloatingPoint type) {
}

@Override public MinorType visit(Timestamp type) {
if (type.getUnit() != TimeUnit.MILLISECOND) {
throw new UnsupportedOperationException("Only milliseconds supported: " + type);
}
return MinorType.TIMESTAMP;
}

Expand Down
3 changes: 2 additions & 1 deletion java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static org.apache.arrow.flatbuf.Precision.SINGLE;
import static org.junit.Assert.assertEquals;

import org.apache.arrow.flatbuf.TimeUnit;
import org.apache.arrow.flatbuf.UnionMode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
Expand Down Expand Up @@ -80,7 +81,7 @@ public void nestedSchema() {
new Field("child4.1", true, Utf8.INSTANCE, null)
)));
childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMP.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
new Field("child5.1", true, new Timestamp("UTC"), null),
new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND), null),
new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
)));
Schema initialSchema = new Schema(childrenBuilder.build());
Expand Down