ARROW-824: Date and Time Vectors should reflect timezone-less semantics

ci/travis_script_integration.sh

@@ -18,7 +18,8 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 pushd $ARROW_JAVA_DIR
 
-mvn package
+echo "mvn package"
+mvn -B clean package 2>&1 > mvn_package.log || (cat mvn_package.log && false)
 
 popd
 

integration/integration_test.py

@@ -544,7 +544,8 @@ def get_json(self):
 
 class JSONFile(object):
 
-    def __init__(self, schema, batches):
+    def __init__(self, name, schema, batches):
+        self.name = name
         self.schema = schema
         self.batches = batches
 
@@ -579,7 +580,7 @@ def get_field(name, type_, nullable=True):
         raise TypeError(dtype)
 
 
-def _generate_file(fields, batch_sizes):
+def _generate_file(name, fields, batch_sizes):
     schema = JSONSchema(fields)
     batches = []
     for size in batch_sizes:
@@ -590,7 +591,7 @@ def _generate_file(fields, batch_sizes):
 
         batches.append(JSONRecordBatch(size, columns))
 
-    return JSONFile(schema, batches)
+    return JSONFile(name, schema, batches)
 
 
 def generate_primitive_case(batch_sizes):
@@ -604,7 +605,7 @@ def generate_primitive_case(batch_sizes):
         fields.append(get_field(type_ + "_nullable", type_, True))
         fields.append(get_field(type_ + "_nonnullable", type_, False))
 
-    return _generate_file(fields, batch_sizes)
+    return _generate_file("primitive", fields, batch_sizes)
 
 
 def generate_datetime_case():
@@ -619,11 +620,11 @@ def generate_datetime_case():
         TimestampType('f7', 'ms'),
         TimestampType('f8', 'us'),
         TimestampType('f9', 'ns'),
-        TimestampType('f10', 'ms', tz='America/New_York')
+        TimestampType('f10', 'ms', tz=None)
     ]
 
     batch_sizes = [7, 10]
-    return _generate_file(fields, batch_sizes)
+    return _generate_file("datetime", fields, batch_sizes)
 
 
 def generate_nested_case():
@@ -637,7 +638,7 @@ def generate_nested_case():
     ]
 
     batch_sizes = [7, 10]
-    return _generate_file(fields, batch_sizes)
+    return _generate_file("nested", fields, batch_sizes)
 
 
 def get_generated_json_files():
@@ -655,7 +656,7 @@ def _temp_path():
 
     generated_paths = []
     for file_obj in file_objs:
-        out_path = os.path.join(temp_dir, guid() + '.json')
+        out_path = os.path.join(temp_dir, 'generated_' + file_obj.name + '.json')
         file_obj.write(out_path)
         generated_paths.append(out_path)
 
@@ -684,20 +685,24 @@ def _compare_implementations(self, producer, consumer):
                                                        consumer.name))
 
         for json_path in self.json_files:
+            print('=====================================================================================')
             print('Testing file {0}'.format(json_path))
+            print('=====================================================================================')
+
+            name = os.path.splitext(os.path.basename(json_path))[0]
 
             # Make the random access file
             print('-- Creating binary inputs')
-            producer_file_path = os.path.join(self.temp_dir, guid())
+            producer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.json_to_arrow')
             producer.json_to_file(json_path, producer_file_path)
 
             # Validate the file
             print('-- Validating file')
             consumer.validate(json_path, producer_file_path)
 
             print('-- Validating stream')
-            producer_stream_path = os.path.join(self.temp_dir, guid())
-            consumer_file_path = os.path.join(self.temp_dir, guid())
+            producer_stream_path = os.path.join(self.temp_dir, guid() + '_' + name + '.arrow_to_stream')
+            consumer_file_path = os.path.join(self.temp_dir, guid() + '_' + name + '.stream_to_arrow')
             producer.file_to_stream(producer_file_path,
                                     producer_stream_path)
             consumer.stream_to_file(producer_stream_path,

java/vector/src/main/codegen/data/ValueVectorTypes.tdd

@@ -59,7 +59,7 @@
         { class: "DateDay" },
         { class: "IntervalYear", javaType: "int", friendlyType: "Period" },
         { class: "TimeSec" },
-        { class: "TimeMilli", javaType: "int", friendlyType: "DateTime" }
+        { class: "TimeMilli", javaType: "int", friendlyType: "LocalDateTime" }
       ]
     },
     {
@@ -71,12 +71,12 @@
       minor: [
         { class: "BigInt"},
         { class: "UInt8" },
-        { class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
-        { class: "DateMilli", javaType: "long", friendlyType: "DateTime" },
-        { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
-        { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
-        { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
-        { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" },
+        { class: "Float8",         javaType: "double", boxedType: "Double",                              fields: [{name: "value", type: "double"}], },
+        { class: "DateMilli",      javaType: "long",                      friendlyType: "LocalDateTime" },
+        { class: "TimeStampSec",   javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "TimeStampMilli", javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "TimeStampMicro", javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
+        { class: "TimeStampNano",  javaType: "long",   boxedType: "Long", friendlyType: "LocalDateTime" },
         { class: "TimeMicro" },
         { class: "TimeNano" }
       ]

java/vector/src/main/codegen/includes/vv_imports.ftl

@@ -57,6 +57,7 @@ import java.math.BigDecimal;
 import java.math.BigInteger;
 
 import org.joda.time.DateTime;
+import org.joda.time.LocalDateTime;
 import org.joda.time.Period;
 
 

java/vector/src/main/codegen/templates/AbstractFieldReader.java

@@ -26,8 +26,8 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
-/*
- * This class is generated using freemarker and the ${.template_name} template.
+/**
+ * Source code generated using FreeMarker template ${.template_name}
  */
 @SuppressWarnings("unused")
 abstract class AbstractFieldReader extends AbstractBaseReader implements FieldReader{
@@ -51,7 +51,7 @@ public Field getField() {
   }
 
   <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
-          "Character", "DateTime", "Period", "Double", "Float",
+          "Character", "LocalDateTime", "Period", "Double", "Float",
           "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>

java/vector/src/main/codegen/templates/ArrowType.java

@@ -38,6 +38,7 @@
 
 /**
  * Arrow types
+ * Source code generated using FreeMarker template ${.template_name}
  **/
 @JsonTypeInfo(
   use = JsonTypeInfo.Id.NAME,

java/vector/src/main/codegen/templates/BaseReader.java

@@ -26,8 +26,9 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
-
-
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public interface BaseReader extends Positionable{
   Field getField();

java/vector/src/main/codegen/templates/ComplexReaders.java

@@ -47,6 +47,9 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public class ${name}ReaderImpl extends AbstractFieldReader {
 
@@ -123,12 +126,16 @@ public Object readObject(){
 package org.apache.arrow.vector.complex.reader;
 
 <#include "/@includes/vv_imports.ftl" />
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public interface ${name}Reader extends BaseReader{
 
   public void read(${minor.class?cap_first}Holder h);
   public void read(Nullable${minor.class?cap_first}Holder h);
   public Object readObject();
+  // read friendly type
   public ${friendlyType} read${safeType}();
   public boolean isSet();
   public void copyAsValue(${minor.class}Writer writer);

java/vector/src/main/codegen/templates/ComplexWriters.java

@@ -139,6 +139,9 @@ public void writeNull() {
 package org.apache.arrow.vector.complex.writer;
 
 <#include "/@includes/vv_imports.ftl" />
+/*
+ * This class is generated using FreeMarker on the ${.template_name} template.
+ */
 @SuppressWarnings("unused")
 public interface ${eName}Writer extends BaseWriter {
   public void write(${minor.class}Holder h);

java/vector/src/main/codegen/templates/FixedValueVectors.java

@@ -495,33 +495,29 @@ public long getTwoAsLong(int index) {
     <#elseif minor.class == "DateMilli">
     @Override
     public ${friendlyType} getObject(int index) {
-      org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
       return date;
     }
 
     <#elseif minor.class == "TimeMilli">
     @Override
     public ${friendlyType} getObject(int index) {
-      org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-      time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
       return time;
     }
 
     <#elseif minor.class == "TimeStampSec">
     @Override
     public ${friendlyType} getObject(int index) {
       long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
-      org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(secs, org.joda.time.DateTimeZone.UTC);
       return date;
     }
 
     <#elseif minor.class == "TimeStampMilli">
     @Override
     public ${friendlyType} getObject(int index) {
-        org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-        date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+        org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
         return date;
     }
 
@@ -530,8 +526,7 @@ public long getTwoAsLong(int index) {
     public ${friendlyType} getObject(int index) {
       // value is truncated when converting microseconds to milliseconds in order to use DateTime type
       long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
-      org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(micros, org.joda.time.DateTimeZone.UTC);
       return date;
     }
 
@@ -540,8 +535,7 @@ public long getTwoAsLong(int index) {
     public ${friendlyType} getObject(int index) {
       // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
       long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
-      org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
-      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(millis, org.joda.time.DateTimeZone.UTC);
       return date;
     }
 

java/vector/src/main/codegen/templates/HolderReaderImpl.java

@@ -84,7 +84,7 @@ public boolean isSet() {
 
   }
 
-@Override
+  @Override
   public void read(${name}Holder h) {
   <#list fields as field>
     h.${field.name} = holder.${field.name};
@@ -99,7 +99,7 @@ public void read(Nullable${name}Holder h) {
     h.isSet = isSet() ? 1 : 0;
   }
 
-
+  // read friendly type
   @Override
   public ${friendlyType} read${safeType}(){
 <#if nullMode == "Nullable">
@@ -114,15 +114,15 @@ public void read(Nullable${name}Holder h) {
       byte[] value = new byte [length];
       holder.buffer.getBytes(holder.start, value, 0, length);
 
-<#if minor.class == "VarBinary">
+  <#if minor.class == "VarBinary">
       return value;
-<#elseif minor.class == "Var16Char">
+  <#elseif minor.class == "Var16Char">
       return new String(value);
-<#elseif minor.class == "VarChar">
+  <#elseif minor.class == "VarChar">
       Text text = new Text();
       text.set(value);
       return text;
-</#if>
+  </#if>
 
 <#elseif minor.class == "Interval">
       Period p = new Period();

java/vector/src/main/codegen/templates/NullReader.java

@@ -29,7 +29,9 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
-
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public class NullReader extends AbstractBaseReader implements FieldReader{
 
@@ -127,7 +129,7 @@ private void fail(String name){
   }
 
   <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean", 
-          "Character", "DateTime", "Period", "Double", "Float",
+          "Character", "LocalDateTime", "Period", "Double", "Float",
           "Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>

java/vector/src/main/codegen/templates/UnionReader.java

@@ -28,7 +28,9 @@
 package org.apache.arrow.vector.complex.impl;
 
 <#include "/@includes/vv_imports.ftl" />
-
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 @SuppressWarnings("unused")
 public class UnionReader extends AbstractFieldReader {
 
@@ -122,7 +124,7 @@ public void copyAsValue(UnionWriter writer) {
   }
 
   <#list ["Object", "Integer", "Long", "Boolean",
-          "Character", "DateTime", "Double", "Float",
+          "Character", "LocalDateTime", "Double", "Float",
           "Text", "Byte", "Short", "byte[]"] as friendlyType>
   <#assign safeType=friendlyType />
   <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>

java/vector/src/main/codegen/templates/UnionVector.java

@@ -50,6 +50,7 @@
  *
  * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
  * each time the vector is accessed.
+ * Source code generated using FreeMarker template ${.template_name}
  */
 public class UnionVector implements FieldVector {
 

java/vector/src/main/codegen/templates/ValueHolders.java

@@ -29,6 +29,9 @@
 
 <#include "/@includes/vv_imports.ftl" />
 
+/**
+ * Source code generated using FreeMarker template ${.template_name}
+ */
 public final class ${className} implements ValueHolder{
 
     <#if mode.name == "Repeated">