From eea8b791123b60a2093e79237d28bbc5b1138231 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Mon, 9 Dec 2019 16:33:43 +0800 Subject: [PATCH 1/3] [ARROW-7301][Java] Sql type DATE should correspond to DateDayVector Change-Id: I604a6b2db3b0816aec6bc7c117e8172507dbdddd --- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 6 +++--- .../adapter/jdbc/consumer/DateConsumer.java | 21 ++++++++++--------- .../adapter/jdbc/JdbcToArrowTestHelper.java | 12 +++++------ .../jdbc/h2/JdbcToArrowDataTypesTest.java | 6 +++--- .../adapter/jdbc/h2/JdbcToArrowNullTest.java | 10 ++++----- .../adapter/jdbc/h2/JdbcToArrowTest.java | 6 +++--- .../jdbc/h2/JdbcToArrowTimeZoneTest.java | 6 +++--- .../h2/JdbcToArrowVectorIteratorTest.java | 14 +++++++------ .../resources/h2/test1_all_datatypes_h2.yml | 2 +- ...t1_all_datatypes_selected_null_rows_h2.yml | 2 +- .../src/test/resources/h2/test1_date_h2.yml | 20 +++++++++--------- .../test/resources/h2/test1_est_date_h2.yml | 20 +++++++++--------- .../test/resources/h2/test1_gmt_date_h2.yml | 20 +++++++++--------- .../test/resources/h2/test1_pst_date_h2.yml | 20 +++++++++--------- .../apache/arrow/vector/DateDayVector.java | 6 ++++++ 15 files changed, 90 insertions(+), 81 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 21278e4cabd..303aa5fbbfb 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -59,7 +59,7 @@ import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.Float4Vector; @@ -265,7 +265,7 @@ public static ArrowType getArrowTypeForJdbcField(JdbcFieldInfo fieldInfo, Calend case Types.CLOB: return new ArrowType.Utf8(); case Types.DATE: - return new ArrowType.Date(DateUnit.MILLISECOND); + return new ArrowType.Date(DateUnit.DAY); case Types.TIME: return new ArrowType.Time(TimeUnit.MILLISECOND, 32); case Types.TIMESTAMP: @@ -402,7 +402,7 @@ static JdbcConsumer getConsumer(ResultSet resultSet, int columnIndex, int jdbcCo case Types.LONGNVARCHAR: return VarCharConsumer.createConsumer((VarCharVector) vector, columnIndex, nullable); case Types.DATE: - return DateConsumer.createConsumer((DateMilliVector) vector, columnIndex, nullable, calendar); + return DateConsumer.createConsumer((DateDayVector) vector, columnIndex, nullable, calendar); case Types.TIME: return TimeConsumer.createConsumer((TimeMilliVector) vector, columnIndex, nullable, calendar); case Types.TIMESTAMP: diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java index 2fbffa08cf7..3e3722c1c3a 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java @@ -22,6 +22,7 @@ import java.util.Calendar; import java.util.Date; +import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DateMilliVector; /** @@ -33,8 +34,8 @@ public class DateConsumer { /** * Creates a consumer for {@link DateMilliVector}. */ - public static JdbcConsumer createConsumer( - DateMilliVector vector, int index, boolean nullable, Calendar calendar) { + public static JdbcConsumer createConsumer( + DateDayVector vector, int index, boolean nullable, Calendar calendar) { if (nullable) { return new NullableDateConsumer(vector, index, calendar); } else { @@ -45,21 +46,21 @@ public static JdbcConsumer createConsumer( /** * Nullable consumer for date. */ - static class NullableDateConsumer extends BaseConsumer { + static class NullableDateConsumer extends BaseConsumer { protected final Calendar calendar; /** * Instantiate a DateConsumer. */ - public NullableDateConsumer(DateMilliVector vector, int index) { + public NullableDateConsumer(DateDayVector vector, int index) { this(vector, index, /* calendar */null); } /** * Instantiate a DateConsumer. */ - public NullableDateConsumer(DateMilliVector vector, int index, Calendar calendar) { + public NullableDateConsumer(DateDayVector vector, int index, Calendar calendar) { super(vector, index); this.calendar = calendar; } @@ -69,7 +70,7 @@ public void consume(ResultSet resultSet) throws SQLException { Date date = calendar == null ? resultSet.getDate(columnIndexInResultSet) : resultSet.getDate(columnIndexInResultSet, calendar); if (!resultSet.wasNull()) { - vector.setSafe(currentIndex, date.getTime()); + vector.setSafe(currentIndex, (int) (date.getTime() / DateDayVector.MILLIS_PER_DAY)); } currentIndex++; } @@ -78,21 +79,21 @@ public void consume(ResultSet resultSet) throws SQLException { /** * Non-nullable consumer for date. */ - static class NonNullableDateConsumer extends BaseConsumer { + static class NonNullableDateConsumer extends BaseConsumer { protected final Calendar calendar; /** * Instantiate a DateConsumer. */ - public NonNullableDateConsumer(DateMilliVector vector, int index) { + public NonNullableDateConsumer(DateDayVector vector, int index) { this(vector, index, /* calendar */null); } /** * Instantiate a DateConsumer. */ - public NonNullableDateConsumer(DateMilliVector vector, int index, Calendar calendar) { + public NonNullableDateConsumer(DateDayVector vector, int index, Calendar calendar) { super(vector, index); this.calendar = calendar; } @@ -101,7 +102,7 @@ public NonNullableDateConsumer(DateMilliVector vector, int index, Calendar calen public void consume(ResultSet resultSet) throws SQLException { Date date = calendar == null ? resultSet.getDate(columnIndexInResultSet) : resultSet.getDate(columnIndexInResultSet, calendar); - vector.setSafe(currentIndex, date.getTime()); + vector.setSafe(currentIndex, (int) (date.getTime() / DateDayVector.MILLIS_PER_DAY)); currentIndex++; } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 3e21144c0ca..c194dfbc355 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -32,7 +32,7 @@ import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; @@ -173,14 +173,14 @@ public static void assertTimeVectorValues(TimeMilliVector timeMilliVector, int r } } - public static void assertDateVectorValues(DateMilliVector dateMilliVector, int rowCount, Long[] values) { - assertEquals(rowCount, dateMilliVector.getValueCount()); + public static void assertDateVectorValues(DateDayVector dateDayVector, int rowCount, Integer[] values) { + assertEquals(rowCount, dateDayVector.getValueCount()); - for (int j = 0; j < dateMilliVector.getValueCount(); j++) { + for (int j = 0; j < dateDayVector.getValueCount(); j++) { if (values[j] == null) { - assertTrue(dateMilliVector.isNull(j)); + assertTrue(dateDayVector.isNull(j)); } else { - assertEquals(values[j].longValue(), dateMilliVector.get(j)); + assertEquals(values[j].longValue(), dateDayVector.get(j)); } } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java index 650c766e02a..cafb7a050d7 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java @@ -50,7 +50,7 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; @@ -202,8 +202,8 @@ public void testDataSets(VectorSchemaRoot root) { table.getCharValues()); break; case DATE: - assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, - table.getLongValues()); + assertDateVectorValues((DateDayVector) root.getVector(table.getVector()), table.getValues().length, + table.getIntValues()); break; case TIME: assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java index 199b821ba19..57fcf566d7d 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java @@ -58,7 +58,7 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; @@ -206,8 +206,8 @@ private void testAllVectorValues(VectorSchemaRoot root) { assertBooleanVectorValues((BitVector) root.getVector(BOOL), table.getRowCount(), getBooleanValues(table.getValues(), BOOL)); - assertDateVectorValues((DateMilliVector) root.getVector(DATE), table.getRowCount(), - getLongValues(table.getValues(), DATE)); + assertDateVectorValues((DateDayVector) root.getVector(DATE), table.getRowCount(), + getIntValues(table.getValues(), DATE)); assertTimeVectorValues((TimeMilliVector) root.getVector(TIME), table.getRowCount(), getLongValues(table.getValues(), TIME)); @@ -242,7 +242,7 @@ public void sqlToArrowTestNullValues(String[] vectors, VectorSchemaRoot root, in assertNullValues((Float8Vector) root.getVector(vectors[6]), rowCount); assertNullValues((Float4Vector) root.getVector(vectors[7]), rowCount); assertNullValues((TimeMilliVector) root.getVector(vectors[8]), rowCount); - assertNullValues((DateMilliVector) root.getVector(vectors[9]), rowCount); + assertNullValues((DateDayVector) root.getVector(vectors[9]), rowCount); assertNullValues((TimeStampVector) root.getVector(vectors[10]), rowCount); assertNullValues((VarBinaryVector) root.getVector(vectors[11]), rowCount); assertNullValues((VarCharVector) root.getVector(vectors[12]), rowCount); @@ -265,7 +265,7 @@ public void sqlToArrowTestSelectedNullColumnsValues(String[] vectors, VectorSche assertNullValues((Float8Vector) root.getVector(vectors[2]), rowCount); assertNullValues((Float4Vector) root.getVector(vectors[3]), rowCount); assertNullValues((TimeMilliVector) root.getVector(vectors[4]), rowCount); - assertNullValues((DateMilliVector) root.getVector(vectors[5]), rowCount); + assertNullValues((DateDayVector) root.getVector(vectors[5]), rowCount); assertNullValues((TimeStampVector) root.getVector(vectors[6]), rowCount); assertNullValues((VarBinaryVector) root.getVector(vectors[7]), rowCount); assertNullValues((VarCharVector) root.getVector(vectors[8]), rowCount); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index dfcdb90be7c..b7d517f26e1 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -79,7 +79,7 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; @@ -202,8 +202,8 @@ public void testDataSets(VectorSchemaRoot root) { assertBooleanVectorValues((BitVector) root.getVector(BOOL), table.getRowCount(), getBooleanValues(table.getValues(), BOOL)); - assertDateVectorValues((DateMilliVector) root.getVector(DATE), table.getRowCount(), - getLongValues(table.getValues(), DATE)); + assertDateVectorValues((DateDayVector) root.getVector(DATE), table.getRowCount(), + getIntValues(table.getValues(), DATE)); assertTimeVectorValues((TimeMilliVector) root.getVector(TIME), table.getRowCount(), getLongValues(table.getValues(), TIME)); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java index 002a1239101..ef2b406d120 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java @@ -37,7 +37,7 @@ import org.apache.arrow.adapter.jdbc.JdbcToArrowUtils; import org.apache.arrow.adapter.jdbc.Table; import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.TimeMilliVector; import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -145,8 +145,8 @@ public void testDataSets(VectorSchemaRoot root) { case EST_DATE: case GMT_DATE: case PST_DATE: - assertDateVectorValues((DateMilliVector) root.getVector(table.getVector()), table.getValues().length, - table.getLongValues()); + assertDateVectorValues((DateDayVector) root.getVector(table.getVector()), table.getValues().length, + table.getIntValues()); break; case EST_TIME: case GMT_TIME: diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java index d041f082b0b..fa75c1cdbaf 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java @@ -38,7 +38,7 @@ import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DateMilliVector; +import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; @@ -51,6 +51,7 @@ import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -66,6 +67,7 @@ public JdbcToArrowVectorIteratorTest(Table table) { super(table); } + @Test @Override public void testJdbcToArrowValues() throws SQLException, IOException { @@ -91,7 +93,7 @@ private void validate(ArrowVectorIterator iterator) throws SQLException, IOExcep List vectorsForChar = new ArrayList<>(); List vectorsForBit = new ArrayList<>(); List vectorsForBool = new ArrayList<>(); - List dateMilliVectors = new ArrayList<>(); + List dateDayVectors = new ArrayList<>(); List timeMilliVectors = new ArrayList<>(); List timeStampVectors = new ArrayList<>(); List decimalVectors = new ArrayList<>(); @@ -115,7 +117,7 @@ private void validate(ArrowVectorIterator iterator) throws SQLException, IOExcep vectorsForChar.add((VarCharVector) root.getVector(CHAR)); vectorsForBit.add((BitVector) root.getVector(BIT)); vectorsForBool.add((BitVector) root.getVector(BOOL)); - dateMilliVectors.add((DateMilliVector) root.getVector(DATE)); + dateDayVectors.add((DateDayVector) root.getVector(DATE)); timeMilliVectors.add((TimeMilliVector) root.getVector(TIME)); timeStampVectors.add((TimeStampVector) root.getVector(TIMESTAMP)); decimalVectors.add((DecimalVector) root.getVector(DECIMAL)); @@ -134,7 +136,7 @@ private void validate(ArrowVectorIterator iterator) throws SQLException, IOExcep assertVarCharVectorValues(vectorsForChar, table.getRowCount(), getCharArray(table.getValues(), CHAR)); assertBitVectorValues(vectorsForBit, table.getRowCount(), getIntValues(table.getValues(), BIT)); assertBooleanVectorValues(vectorsForBool, table.getRowCount(), getBooleanValues(table.getValues(), BOOL)); - assertDateMilliVectorValues(dateMilliVectors, table.getRowCount(), getLongValues(table.getValues(), DATE)); + assertDateDayVectorValues(dateDayVectors, table.getRowCount(), getLongValues(table.getValues(), DATE)); assertTimeMilliVectorValues(timeMilliVectors, table.getRowCount(), getLongValues(table.getValues(), TIME)); assertTimeStampVectorValues(timeStampVectors, table.getRowCount(), getLongValues(table.getValues(), TIMESTAMP)); assertDecimalVectorValues(decimalVectors, table.getRowCount(), getDecimalValues(table.getValues(), DECIMAL)); @@ -205,12 +207,12 @@ private void assertTimeMilliVectorValues(List vectors, int rowC } } - private void assertDateMilliVectorValues(List vectors, int rowCount, Long[] values) { + private void assertDateDayVectorValues(List vectors, int rowCount, Long[] values) { int valueCount = vectors.stream().mapToInt(ValueVector::getValueCount).sum(); assertEquals(rowCount, valueCount); int index = 0; - for (DateMilliVector vector : vectors) { + for (DateDayVector vector : vectors) { for (int i = 0; i < vector.getValueCount(); i++) { assertEquals(values[index++].longValue(), vector.get(i)); } diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml index fc9784df7ff..9baae643aef 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_h2.yml @@ -85,7 +85,7 @@ values: - 'DECIMAL_FIELD6=17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23,17345667789.23' - 'DOUBLE_FIELD7=56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345,56478356785.345' - 'TIME_FIELD9=45935000,45935000,45935000,45935000,45935000,45935000,45935000,45935000,45935000,45935000' - - 'DATE_FIELD10=1518393600000,1518393600000,1518393600000,1518393600000,1518393600000,1518393600000,1518393600000,1518393600000,1518393600000,1518393600000' + - 'DATE_FIELD10=17574,17574,17574,17574,17574,17574,17574,17574,17574,17574' - 'TIMESTAMP_FIELD11=1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000,1518439535000' - 'CHAR_FIELD16=some char text,some char text,some char text,some char text,some char text, some char text,some char text,some char text,some char text,some char text' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_selected_null_rows_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_selected_null_rows_h2.yml index bb9bfb81548..4be8ab86ecc 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_selected_null_rows_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_all_datatypes_selected_null_rows_h2.yml @@ -71,7 +71,7 @@ values: - 'DECIMAL_FIELD6=null,17345667789.23,null,17345667789.23,null' - 'DOUBLE_FIELD7=null,56478356785.345,null,56478356785.345,null' - 'TIME_FIELD9=null,45935000,null,45935000,null' - - 'DATE_FIELD10=null,1518393600000,null,1518393600000,null' + - 'DATE_FIELD10=null,17574,null,17574,null' - 'TIMESTAMP_FIELD11=null,1518439535000,null,1518439535000,null' - 'CHAR_FIELD16=null,some char text,null,some char text,null' - 'VARCHAR_FIELD13=null,some text that needs to be converted to varchar,null, diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml index 45aa56c7417..da33d9d47cd 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_date_h2.yml @@ -34,13 +34,13 @@ query: 'select date_field10 from table1;' drop: 'DROP table table1;' values: - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_est_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_est_date_h2.yml index 290c32ebafa..1868db3adeb 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_est_date_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_est_date_h2.yml @@ -36,13 +36,13 @@ query: 'select date_field10 from table1;' drop: 'DROP table table1;' values: - - '1518411600000' - - '1518411600000' - - '1518411600000' - - '1518411600000' - - '1518411600000' - - '1518411600000' - - '1518411600000' - - '1518411600000' - - '1518411600000' - - '1518411600000' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_gmt_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_gmt_date_h2.yml index 03929c936e4..65824861a8a 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_gmt_date_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_gmt_date_h2.yml @@ -36,13 +36,13 @@ query: 'select date_field10 from table1;' drop: 'DROP table table1;' values: - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' - - '1518393600000' \ No newline at end of file + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' \ No newline at end of file diff --git a/java/adapter/jdbc/src/test/resources/h2/test1_pst_date_h2.yml b/java/adapter/jdbc/src/test/resources/h2/test1_pst_date_h2.yml index 81a668f37a4..798cfc7d67d 100644 --- a/java/adapter/jdbc/src/test/resources/h2/test1_pst_date_h2.yml +++ b/java/adapter/jdbc/src/test/resources/h2/test1_pst_date_h2.yml @@ -36,13 +36,13 @@ query: 'select date_field10 from table1;' drop: 'DROP table table1;' values: - - '1518422400000' - - '1518422400000' - - '1518422400000' - - '1518422400000' - - '1518422400000' - - '1518422400000' - - '1518422400000' - - '1518422400000' - - '1518422400000' - - '1518422400000' \ No newline at end of file + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' + - '17574' \ No newline at end of file diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java index 7f244bfa658..19fe5e33cb0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java @@ -37,6 +37,12 @@ * maintained to track which elements in the vector are null. */ public final class DateDayVector extends BaseFixedWidthVector { + + /** + * The number of milli-seconds in a day. + */ + public static final long MILLIS_PER_DAY = 3600 * 24 * 1000L; + private static final byte TYPE_WIDTH = 4; private final FieldReader reader; From be731925db2920790f9a5e3373437c8ec1a15809 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Fri, 27 Dec 2019 10:15:25 +0800 Subject: [PATCH 2/3] [ARROW-7301][Java] Resolve comments --- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 2 +- .../adapter/jdbc/consumer/DateConsumer.java | 34 +++++++++++++++++-- .../apache/arrow/vector/DateDayVector.java | 5 --- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index 303aa5fbbfb..f7bf3248eb4 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -210,7 +210,7 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, JdbcToArrowConfig *
  • BINARY --> ArrowType.Binary
    • *
  • VARBINARY --> ArrowType.Binary
    • *
  • LONGVARBINARY --> ArrowType.Binary
    • - *
  • DATE --> ArrowType.Date(DateUnit.MILLISECOND)
    • + *
  • DATE --> ArrowType.Date(DateUnit.DAY)
    • *
  • TIME --> ArrowType.Time(TimeUnit.MILLISECOND, 32)
    • *
  • TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, calendar timezone)
    • *
  • CLOB --> ArrowType.Utf8
    • diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java index 3e3722c1c3a..c9461b18f44 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java @@ -17,10 +17,13 @@ package org.apache.arrow.adapter.jdbc.consumer; +import java.sql.Date; import java.sql.ResultSet; import java.sql.SQLException; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Calendar; -import java.util.Date; +import java.util.concurrent.TimeUnit; import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DateMilliVector; @@ -31,6 +34,23 @@ */ public class DateConsumer { + /** + * The number of milli-seconds in a day. + */ + public static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1); + + public static final int MAX_DAY; + + static { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + try { + java.util.Date date = dateFormat.parse("9999-12-31"); + MAX_DAY = (int) (date.getTime() / MILLIS_PER_DAY); + } catch (ParseException e) { + throw new IllegalArgumentException("Failed to parse max day", e); + } + } + /** * Creates a consumer for {@link DateMilliVector}. */ @@ -70,7 +90,11 @@ public void consume(ResultSet resultSet) throws SQLException { Date date = calendar == null ? resultSet.getDate(columnIndexInResultSet) : resultSet.getDate(columnIndexInResultSet, calendar); if (!resultSet.wasNull()) { - vector.setSafe(currentIndex, (int) (date.getTime() / DateDayVector.MILLIS_PER_DAY)); + int day = (int) (date.getTime() / MILLIS_PER_DAY); + if (day < 0 || day > MAX_DAY) { + throw new IllegalArgumentException("Day overflow: " + day); + } + vector.setSafe(currentIndex, day); } currentIndex++; } @@ -102,7 +126,11 @@ public NonNullableDateConsumer(DateDayVector vector, int index, Calendar calenda public void consume(ResultSet resultSet) throws SQLException { Date date = calendar == null ? resultSet.getDate(columnIndexInResultSet) : resultSet.getDate(columnIndexInResultSet, calendar); - vector.setSafe(currentIndex, (int) (date.getTime() / DateDayVector.MILLIS_PER_DAY)); + int day = (int) (date.getTime() / MILLIS_PER_DAY); + if (day < 0 || day > MAX_DAY) { + throw new IllegalArgumentException("Day overflow: " + day); + } + vector.setSafe(currentIndex, day); currentIndex++; } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java index 19fe5e33cb0..9448a1a1453 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DateDayVector.java @@ -38,11 +38,6 @@ */ public final class DateDayVector extends BaseFixedWidthVector { - /** - * The number of milli-seconds in a day. - */ - public static final long MILLIS_PER_DAY = 3600 * 24 * 1000L; - private static final byte TYPE_WIDTH = 4; private final FieldReader reader; From a6de377564e529c07896d9c9e673afad7fb61f60 Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Fri, 17 Jan 2020 20:15:01 +0800 Subject: [PATCH 3/3] [ARROW-7301][Java] Remove division in time conversion --- .../arrow/adapter/jdbc/consumer/DateConsumer.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java index c9461b18f44..bb9ab1cf0b9 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java @@ -34,18 +34,13 @@ */ public class DateConsumer { - /** - * The number of milli-seconds in a day. - */ - public static final long MILLIS_PER_DAY = TimeUnit.DAYS.toMillis(1); - public static final int MAX_DAY; static { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); try { java.util.Date date = dateFormat.parse("9999-12-31"); - MAX_DAY = (int) (date.getTime() / MILLIS_PER_DAY); + MAX_DAY = (int) TimeUnit.MILLISECONDS.toDays(date.getTime()); } catch (ParseException e) { throw new IllegalArgumentException("Failed to parse max day", e); } @@ -90,7 +85,7 @@ public void consume(ResultSet resultSet) throws SQLException { Date date = calendar == null ? resultSet.getDate(columnIndexInResultSet) : resultSet.getDate(columnIndexInResultSet, calendar); if (!resultSet.wasNull()) { - int day = (int) (date.getTime() / MILLIS_PER_DAY); + int day = (int) TimeUnit.MILLISECONDS.toDays(date.getTime()); if (day < 0 || day > MAX_DAY) { throw new IllegalArgumentException("Day overflow: " + day); } @@ -126,7 +121,7 @@ public NonNullableDateConsumer(DateDayVector vector, int index, Calendar calenda public void consume(ResultSet resultSet) throws SQLException { Date date = calendar == null ? resultSet.getDate(columnIndexInResultSet) : resultSet.getDate(columnIndexInResultSet, calendar); - int day = (int) (date.getTime() / MILLIS_PER_DAY); + int day = (int) TimeUnit.MILLISECONDS.toDays(date.getTime()); if (day < 0 || day > MAX_DAY) { throw new IllegalArgumentException("Day overflow: " + day); }